Skip to content

Instantly share code, notes, and snippets.

@ngoclamtran
Created Nov 1, 2021
Embed
What would you like to do?
This file has been truncated, but you can view the full file.
goroutine 1386454 [running]:
runtime/pprof.writeGoroutineStacks({0x235d7a0, 0xc00038a620})
/usr/local/go/src/runtime/pprof/pprof.go:693 +0x70
runtime/pprof.writeGoroutine({0x235d7a0, 0xc00038a620}, 0x200)
/usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b
runtime/pprof.(*Profile).WriteTo(0x1e1e9c0, {0x235d7a0, 0xc00038a620}, 0xc)
/usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b
net/http/pprof.handler.ServeHTTP({0xc001ea40d1, 0x34934e0}, {0x2386300, 0xc00038a620}, 0xc001ea40c4)
/usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a
net/http/pprof.Index({0x2386300, 0xc00038a620}, 0xc00016e300)
/usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e
net/http.HandlerFunc.ServeHTTP(0x0, {0x2386300, 0xc00038a620}, 0x0)
/usr/local/go/src/net/http/server.go:2046 +0x2f
net/http.(*ServeMux).ServeHTTP(0xc001ea40db, {0x2386300, 0xc00038a620}, 0xc00016e300)
/usr/local/go/src/net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xc001a3d2f0}, {0x2386300, 0xc00038a620}, 0xc00016e300)
/usr/local/go/src/net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc000dee140, {0x23922d0, 0xc000586750})
/usr/local/go/src/net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:3033 +0x4e8
goroutine 1 [chan receive, 7 minutes]:
go.temporal.io/server/temporal.(*Server).Start(0xc00051e000)
/temporal/temporal/server.go:272 +0xe9c
main.buildCLI.func2(0xc00053dcc0)
/temporal/cmd/server/main.go:168 +0xc4d
github.com/urfave/cli/v2.(*Command).Run(0xc000485440, 0xc00053dc80)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/command.go:163 +0x64a
github.com/urfave/cli/v2.(*App).RunContext(0xc0005311e0, {0x2392260, 0xc000040040}, {0xc00003a080, 0x8, 0x8})
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:313 +0x81e
github.com/urfave/cli/v2.(*App).Run(...)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:224
main.main()
/temporal/cmd/server/main.go:50 +0x45
goroutine 8 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0000b6300)
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92
goroutine 9 [syscall, 7 minutes]:
os/signal.signal_recv()
/usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
/usr/local/go/src/os/signal/signal.go:151 +0x2c
goroutine 10 [chan receive, 7 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 11 [chan receive, 7 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 97 [sleep]:
time.Sleep(0x306dc4200)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 50 [select]:
database/sql.(*DB).connectionOpener(0xc0008efe10, {0x2392228, 0xc000544c80})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 47 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00011cf00)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 67 [select]:
database/sql.(*DB).connectionCleaner(0xc0008efe10, 0xc00011cf00)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 73 [select]:
database/sql.(*DB).connectionOpener(0xc0005beb60, {0x2392228, 0xc000667900})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 37 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf23d69f0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000b7a00, 0x416f26, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0000b7a00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0000b7a00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000c078)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000c078)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc00038a380, {0x23860f0, 0xc00000c078})
/usr/local/go/src/net/http/server.go:3001 +0x394
net/http.(*Server).ListenAndServe(0xc00038a380)
/usr/local/go/src/net/http/server.go:2930 +0x7d
net/http.ListenAndServe(...)
/usr/local/go/src/net/http/server.go:3184
go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start.func1()
/temporal/common/pprof/pprof.go:74 +0x245
created by go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start
/temporal/common/pprof/pprof.go:72 +0x9c
goroutine 194 [select, 2 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0006a4e60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 1073655 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a558, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0020ac100, 0xc002168000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0020ac100, {0xc002168000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0020ac100, {0xc002168000, 0x1060100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f918, {0xc002168000, 0x0, 0x40ce34})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000847d40, {0xc000792740, 0x9, 0x18})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000847d40}, {0xc000792740, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000792740, 0x9, 0xc00003e8d0}, {0x23585c0, 0xc000847d40})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000792700)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00258ef00, 0xc0011fdf98, 0x1418c25)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000183880, {0x23d1088, 0xc00258ef00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185
goroutine 79 [select]:
database/sql.(*DB).connectionCleaner(0xc0005beb60, 0xc000593720)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 1059764 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000412620, {0x2392228, 0xc001053140})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 57 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d6ad8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841b00, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000841b00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000841b00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000095a58)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000095a58)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0000854e0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000690600)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 58 [select, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc0000da820, {0x2392228, 0xc00011b140})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 661066 [select, 3 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc002005220)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 70 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d6908, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841f80, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000841f80)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000841f80)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000095dd0)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000095dd0)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc000182fc0, {0x23860f0, 0xc000095dd0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/history.(*Service).Start(0xc000850fd0)
/temporal/service/history/service.go:99 +0x185
go.temporal.io/server/service/history.ServiceLifetimeHooks.func1.1({0x237c7c8, 0xc0005935e0}, 0x0)
/temporal/service/history/fx.go:170 +0x2c
created by go.temporal.io/server/service/history.ServiceLifetimeHooks.func1
/temporal/service/history/fx.go:168 +0x9a
goroutine 71 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000593720)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 1245921 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000f2a5f0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 63 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 80 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 83 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d6738, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003b200, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003b200)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003b200)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000cbd0)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000cbd0)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000574860)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000702200)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 86 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0006813b0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 85 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d6650, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003b500, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003b500)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003b500)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000ced0)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000ced0)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0003fbc00, {0x23860f0, 0xc00000ced0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/matching.(*Service).Start(0xc000606fd0)
/temporal/service/matching/service.go:128 +0x154
go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1.1({0x237c840, 0xc00068e990}, 0xc00082001c)
/temporal/service/matching/fx.go:131 +0x2c
created by go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1
/temporal/service/matching/fx.go:129 +0x9a
goroutine 110 [select, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc000b245b0, {0x2392228, 0xc0006a1800})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 88 [select, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc00083b5f0, {0x2392228, 0xc0007d6bc0})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 142 [sleep]:
time.Sleep(0x306dc4200)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 106 [select]:
database/sql.(*DB).connectionCleaner(0xc00083b5f0, 0x1c)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 109 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d60e0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0006d0a00, 0x532ca5, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0006d0a00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0006d0a00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0006c8870)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0006c8870)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000575040)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000702400)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 93 [chan receive]:
github.com/rcrowley/go-metrics.(*meterArbiter).tick(0x34906a0)
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:239 +0x2a
created by github.com/rcrowley/go-metrics.NewMeter
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:46 +0xd8
goroutine 94 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 95 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 319 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1250260 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ffefc0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001647970, {0x1f7b7c0, 0xc0015c20a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xcf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73d00, 0xc001747bf8, 0xc001747bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73d00, {0x1f7b7c0, 0xc0015c20a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x20, {0xc0016f7660, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0xc000715b80, {0xc0016f7660, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0xc00218fb70, 0xc000f9f710, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x8, {0xc000756a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0017d9f00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x33, 0xc000f9f650, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0017d9f00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0017d9f00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0017d9f00}, 0xc00249f6e0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc000f9ef90}, 0xc0017d9e80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc000f9ef90}, 0xc0017d9e80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001748480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001748528, {0x235a640, 0xc0006dc210}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21864e8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc000f9ef90}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c000fa0e00})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001748778, {0x235a640, 0xc0000a9fb0}, 0x392d316135342d32)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc000f9ef90}, 0xc0017d9b80)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000c0adb8, 0x4c1cb4}, {0xc057f6a228813d20, 0x6118cdc4f8}, {0xc000c0adb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc000f9ef90}, 0xc0017d9b80)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80}, 0xc0016f7460, 0xc001efe258)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80}, 0x15, 0xc0017d9bc0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80}, 0x2076951, 0xc0017d9bc0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80}, 0x237bc38, 0xc0017d9bc0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000f9ee40}, {0x1f33320, 0xc0017d9b80}, 0x1e5a940, 0xc0017d9bc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ee40}, {0x1f33320, 0xc0017d9b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80}, 0x23c27b0, 0xc0017d9bc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00296db60, {0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80}, 0x40, 0xc00005ec00)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80}, 0xc0016f7460, 0xc001efe258)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc000f9ee10}, 0xc0025f4b40, 0xc0005754c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00083cfc0, 0xc000cf2e10, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00083cfc0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 404 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000472b00)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 184 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf2207658, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000afbe00, 0xc000aeb950, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000afbe00, {0xc000aeb950, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000afbe00, {0xc000aeb950, 0xc00085af50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000663498, {0xc000aeb950, 0x46b213, 0x617f6333})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000663498}, {0xc000aeb950, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006fc2c0, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 320 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 1250263 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000b43f0, {0x2392228, 0xc0027548c0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 211 [chan receive, 7 minutes]:
go.temporal.io/server/service/worker.(*Service).Start(0x1000000010000)
/temporal/service/worker/service.go:171 +0x228
go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1.1({0x237c890, 0xc0000880a0}, 0x0)
/temporal/service/worker/fx.go:80 +0x2c
created by go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1
/temporal/service/worker/fx.go:78 +0x9a
goroutine 147 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 148 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 1242817 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008e97a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac370, {0x1f7c860, 0xc001bac2c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73e00, 0xc001c975e8, 0xc001c975b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73e00, {0x1f7c860, 0xc001bac2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001f1faa0}, {0x20768da, 0x8}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x1c75901, {0xc0016f67c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x7f6c191ae108, {0xc0016f67c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x20, 0xc00078a7b0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0xc000da4820, {0xc001faa7c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x0, 0xc00078a780, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x0, {0xc000574620, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x1e0cfc0, 0xc00078a6f0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x60, {0xc000574620, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001f1faa0}, {0x20768da, 0x203000}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x8, 0xc00078a6c0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x0, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc001f1faa0}, {0x20768da, 0x0}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc001f1faa0}, 0xc000600720, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00166f600, {0x2392298, 0xc001f1faa0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1242762 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001e289c0, {0x2392298, 0xc001f1ef60}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001789d40, {0x23922d0, 0xc00073ae10}, 0x2a)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002618b70, {0x23922d0, 0xc00073ae10}, 0xc000bd2ba0, 0x1, 0xbd2bb0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0007bbdd0, 0xc000edd840)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0007ba480}, 0xc000edd840)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0007ba480}, {0x1f35540, 0xc000edd840})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0007ba480}, {0x1f35540, 0xc000edd840}, 0x237bc38, 0xc001d475c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007ba480}, {0x1f35540, 0xc000edd840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0007f75b0, {0x23922d0, 0xc000839380}, {0x1f35540, 0xc000edd840}, 0xc0010efb40, 0xc000edd8c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000839380}, {0x1f35540, 0xc000edd840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000839380}, {0x1f35540, 0xc000edd840}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000839380}, {0x1f35540, 0xc000edd840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840}, 0x40, 0xc000edd8c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840}, 0xc0016f6600, 0xc001d475c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000839260}, 0xc001f1ef00, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0017887e0, 0xc00070a240, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0017887e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1250194 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001855320)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac630, {0x1f7b560, 0xc000defa40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3d200, 0xc00221fd18, 0xc00221fce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3d200, {0x1f7b560, 0xc000defa40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000c23080}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x20, {0xc000ba13a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000c23080}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c23080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x7f6bf21ba780, {0xc000ba13a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c23080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0xc00218fb70, 0xc000c230b0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c23080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x8, {0xc000756a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001eb31c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x237ac70, 0xc000c22ff0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001eb31c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001eb31c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001eb31c0}, 0xc0016ff140, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc000c22930}, 0xc001eb3100, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc000c22930}, 0xc001eb3100, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0022205a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc002220648, {0x235a640, 0xc0006dc210}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf218ede8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc000c22930}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00206ef40})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc002220898, {0x235a640, 0xc0000a9fb0}, 0x622d343166342d36)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc000c22930}, 0xc000c22780)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001808db8, 0x4c1cb4}, {0xc057f6a2279fe76f, 0x6117ec6f39}, {0xc001808db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc000c22930}, 0xc000c22780)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780}, 0xc000ba1240, 0xc000899440)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780}, 0x15, 0xc001eb2ec0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780}, 0x207690b, 0xc001eb2ec0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780}, 0x237bc38, 0xc001eb2ec0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000c227e0}, {0x1f33180, 0xc000c22780}, 0x1e5a940, 0xc001eb2ec0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c227e0}, {0x1f33180, 0xc000c22780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780}, 0x9, 0xc001eb2ec0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780}, 0xc0007f7768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780}, 0x40, 0xc00005ec00)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780}, 0xc000ba1240, 0xc000899440)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc000c22750}, 0xc0021cb860, 0xc0005754c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001854000, 0xc000cf2e10, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001854000, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 401 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00040c420)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 318 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 189 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 1254954 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0022d7680, {0x2392228, 0xc001fabb40})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1250117 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00077f320)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001a0f600, {0x1f7c860, 0xc001a0ea50}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9800, 0xc000ee35e8, 0xc000ee35b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9800, {0x1f7c860, 0xc001a0ea50})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00275c000}, {0x20768da, 0x8}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x1c75901, {0xc001880f60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x7f6c191ae5b8, {0xc001880f60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x20, 0xc0016b3320, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0xc000da4820, {0xc000ffda80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x0, 0xc0016b32f0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x0, {0xc000574620, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x1e0cfc0, 0xc0016b3260, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x60, {0xc000574620, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00275c000}, {0x20768da, 0x203000}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x8, 0xc0016b3230, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x7, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae5b8, {0x2392298, 0xc00275c000}, {0x20768da, 0x0}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc00275c000}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00003b480, {0x2392298, 0xc00275c000})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1254950 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d2ea20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc0630, {0x1f7b560, 0xc0015c3220}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3da00, 0xc0019ce440, 0xc0019ce410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3da00, {0x1f7b560, 0xc0015c3220})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00078a720}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x20, {0xc0013e8b00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00078a720}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00078a720}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x7f6bf23dd368, {0xc0013e8b00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00078a720}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0xc00218ff30, 0xc00078a750, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00078a720}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x8, {0xc000ed4b60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001fabc80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x237ac70, 0xc00078a690, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001fabc80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001fabc80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc001fabc80}, 0xc000594ea6, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc0022d7aa0}, 0xc001fabc00, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc0022d7aa0}, 0xc001fabc00, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0022d77a0, {0x2392298, 0xc0022d7aa0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0022d7a40, {0x2392298, 0xc0022d7aa0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001d2e120, {0x23922d0, 0xc000784e10}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc000784e10}, 0xc0007573a0, 0x1bd58c0, 0x7573c0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc000784db0, 0xc001fab140)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc000784b10}, 0xc001fab140)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000784b10}, {0x1f353a0, 0xc001fab140})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc000784b10}, {0x1f353a0, 0xc001fab140}, 0x237bc38, 0xc00086c858)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000784b10}, {0x1f353a0, 0xc001fab140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00066adb0, {0x23922d0, 0xc00066ade0}, {0x1f353a0, 0xc001fab140}, 0xc0007a1950, 0xc001fab1c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00066ade0}, {0x1f353a0, 0xc001fab140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00066ade0}, {0x1f353a0, 0xc001fab140}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00066ade0}, {0x1f353a0, 0xc001fab140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140}, 0x40, 0xc001fab1c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140}, 0xc0013e89c0, 0xc00086c858)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00066ad50}, 0xc0022d7440, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c51200, 0xc00070a240, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c51200, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 185 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006fc2c0, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 337 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730210)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 338 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730000)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 1270884 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001be10e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbc840, {0x1f736c0, 0xc00132cea0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8b00, 0xc0015af5c0, 0xc0015af590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8b00, {0x1f736c0, 0xc00132cea0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000efc8a0}, {0x2076920, 0x8}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x1c75901, {0xc001b58400, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x7f6c191ae108, {0xc001b58400, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x20, 0xc00133d4d0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0xc000d3fa00, {0xc001327d40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x0, 0xc00133d4a0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x0, {0xc000574620, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x0, 0xc00133d410, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x40ce34, {0xc000574620, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000efc8a0}, {0x2076920, 0x203000}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x203000, 0xc00133d3e0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x7f6c191ae108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002994e50, {0x2392298, 0xc000efc8a0}, {0x2076920, 0x200efa298}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc000efc8a0}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e89c70, {0x2392298, 0xc000efc8a0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 587 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00040def0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 366 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1270930 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0018f1d40, {0x2392298, 0xc000ebe420}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0019e0900, {0x23922d0, 0xc0015ed4a0}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001898240, {0x23922d0, 0xc0015ed4a0}, 0xc0015d4d20, 0x1, 0x15d4d30)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0015ed440, 0xc0015d6e40)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0015ed1d0}, 0xc0015d6e40)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0015ed1d0}, {0x1f35540, 0xc0015d6e40})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0015ed1d0}, {0x1f35540, 0xc0015d6e40}, 0x237bc38, 0xc002423440)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed1d0}, {0x1f35540, 0xc0015d6e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0015ed080}, {0x1f35540, 0xc0015d6e40}, 0xc0015d6500, 0xc0015d6ec0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed080}, {0x1f35540, 0xc0015d6e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0015ed080}, {0x1f35540, 0xc0015d6e40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed080}, {0x1f35540, 0xc0015d6e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40}, 0x40, 0xc0015d6ec0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40}, 0xc001b59ca0, 0xc002423440)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0015ed050}, 0xc000ebe3c0, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008a9560, 0xc00070a240, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008a9560, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1255012 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002663aa0, {0x2392228, 0xc001fe3900})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 271 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf22071d0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0005adb80, 0xc00051c640, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0005adb80, {0xc00051c640, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0005adb80, {0xc00051c640, 0xc000cb6f00, 0xc0005782a0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000096798, {0xc00051c640, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000096798}, {0xc00051c640, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00072d080, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 581 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 269 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf2207000, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841900, 0xc00051c530, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000841900, {0xc00051c530, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000841900, {0xc00051c530, 0xc000b90f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000967a0, {0xc00051c530, 0x46b213, 0x617f6332})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000967a0}, {0xc00051c530, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00072cdc0, 0x678630)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 270 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00072cdc0, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1271024 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 582 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 181 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf22073a0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000afbf80, 0xc000aeb760, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000afbf80, {0xc000aeb760, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000afbf80, {0xc000aeb760, 0xc0015d0960, 0xc000756c70})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0006634a0, {0xc000aeb760, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0006634a0}, {0xc000aeb760, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006fc000, 0xbd3220)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 182 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006fc000, 0x690600)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 193 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf23d5ff8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0006d0d00, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0006d0d00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0006d0d00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0006c8b70)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0006c8b70)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc000183880, {0x23860f0, 0xc0006c8b70})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/frontend.(*Service).Start(0xc000af5fd0)
/temporal/service/frontend/service.go:238 +0x230
go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1.1({0x237c750, 0xc0006d0080}, 0x0)
/temporal/service/frontend/fx.go:284 +0x2c
created by go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1
/temporal/service/frontend/fx.go:282 +0x9a
goroutine 197 [select, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc000b25a00, {0x2392228, 0xc000544d40})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 190 [select]:
database/sql.(*DB).connectionCleaner(0xc000b25a00, 0x1fef77d)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 209 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf2207910, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce0680, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000ce0680)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000ce0680)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0008cdba8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0008cdba8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000ba0c40)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000690200)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 212 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 204 [sleep]:
time.Sleep(0x37e11d600)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 1254955 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002016c00, {0x2392298, 0xc0022d7e60}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005f37a0, {0x23922d0, 0xc000752390}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc000752390}, 0xc000ea1020, 0x1bd58c0, 0xea1040)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc000636990, 0xc001fabf80)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc000636570}, 0xc001fabf80)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000636570}, {0x1f353a0, 0xc001fabf80})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc000636570}, {0x1f353a0, 0xc001fabf80}, 0x237bc38, 0xc00086cd98)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000636570}, {0x1f353a0, 0xc001fabf80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0006363f0}, {0x1f353a0, 0xc001fabf80}, 0x203000, 0xc001fabfc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006363f0}, {0x1f353a0, 0xc001fabf80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0006363f0}, {0x1f353a0, 0xc001fabf80}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006363f0}, {0x1f353a0, 0xc001fabf80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80}, 0x40, 0xc001fabfc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80}, 0xc0013e8d40, 0xc00086cd98)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000636390}, 0xc0022d7e00, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001d2eb40, 0xc00070a240, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001d2eb40, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 465 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730370)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 218 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 225 [sleep]:
time.Sleep(0x37e11d600)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 217 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 1049 [select, 7 minutes]:
net/http.(*persistConn).readLoop(0xc0010fd7a0)
/usr/local/go/src/net/http/transport.go:2207 +0xd8a
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1747 +0x1e05
goroutine 272 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00072d080, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 340 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730160)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 339 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007300b0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 367 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 368 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 661067 [select, 3 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000fe4d0, {0x2392228, 0xc002042200})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 586 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0004728f0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 402 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0004729a0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 403 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000472a50)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 1254935 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0016d2000, {0x2392228, 0xc000966c80})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 266 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a8f8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841880, 0xc0005409b0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000841880, {0xc0005409b0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000841880, {0xc0005409b0, 0xc001f97920, 0xc0007574a0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000096778, {0xc0005409b0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000096778}, {0xc0005409b0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00061c000, 0x6782f0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 267 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00061c000, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 229 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a9e0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841800, 0xc0003d5be0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000841800, {0xc0003d5be0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000841800, {0xc0003d5be0, 0xc000903f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000096730, {0xc0003d5be0, 0x46b213, 0x617f6333})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000096730}, {0xc0003d5be0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000700840, 0x1fe0001)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 230 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000700840, 0x1c)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 301 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000584580, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 232 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 233 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 300 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221ad80, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce1800, 0xc000616040, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ce1800, {0xc000616040, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ce1800, {0xc000616040, 0xc000900f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f118, {0xc000616040, 0x46b213, 0x617f6333})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f118}, {0xc000616040, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000584580, 0x5ba530)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 583 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 466 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730420)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 304 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a2a0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce1700, 0xc0006161b0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ce1700, {0xc0006161b0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ce1700, {0xc0006161b0, 0xc000f22a80, 0xc0005784f0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f0f0, {0xc0006161b0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f0f0}, {0xc0006161b0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000584b00, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 398 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1245925 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002166d80, {0x2392298, 0xc00247a240}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0010fc6c0, {0x23922d0, 0xc002409230}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002409230}, 0xc00184c740, 0x1bd58c0, 0x184c750)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc002409020, 0xc000f27340)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc002408d80}, 0xc000f27340)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002408d80}, {0x1f353a0, 0xc000f27340})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc002408d80}, {0x1f353a0, 0xc000f27340}, 0x237bc38, 0xc0022f85a0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408d80}, {0x1f353a0, 0xc000f27340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002408c30}, {0x1f353a0, 0xc000f27340}, 0x203000, 0xc000f27380)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408c30}, {0x1f353a0, 0xc000f27340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002408c30}, {0x1f353a0, 0xc000f27340}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408c30}, {0x1f353a0, 0xc000f27340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340}, 0x40, 0xc000f27380)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340}, 0xc0013e9540, 0xc0022f85a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc002408bd0}, 0xc00247a1e0, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc000886c60, 0xc00070a240, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc000886c60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1245961 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00177c360)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0016460b0, {0x1f7b560, 0xc00068a320}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036600, 0xc001605d18, 0xc001605ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036600, {0x1f7b560, 0xc00068a320})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x20, {0xc0013e99a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0xc001852840, {0xc0013e99a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0xc00218fb70, 0xc00132fe00, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x8, {0xc000756a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000f38d40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x237ac70, 0xc00132fd40, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000f38d40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000f38d40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000f38d40}, 0xc001ba8c00, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc00132f500}, 0xc000f38cc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc00132f500}, 0xc000f38cc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0016065a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001606648, {0x235a640, 0xc0006dc210}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf20d7388, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc00132f500}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00189eb80})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001606898, {0x235a640, 0xc0000a9fb0}, 0x612d313030342d36)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc00132f500}, 0xc00132f350)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001caadb8, 0x4c1cb4}, {0xc057f6a1ea49df62, 0x60defb9d2e}, {0xc001caadb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc00132f500}, 0xc00132f350)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350}, 0xc0013e98e0, 0xc0022f9560)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350}, 0x15, 0xc000f38a80)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350}, 0x207690b, 0xc000f38a80)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350}, 0x237bc38, 0xc000f38a80)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00132f3b0}, {0x1f33180, 0xc00132f350}, 0x1e5a940, 0xc000f38a80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f3b0}, {0x1f33180, 0xc00132f350})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350}, 0x9, 0xc000f38a80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350}, 0xc002993768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002993b60, {0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350}, 0x40, 0xc00005e400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350}, 0xc0013e98e0, 0xc0022f9560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00132f320}, 0xc00247afc0, 0xc0005754c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000830480, 0xc000cf2e10, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000830480, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 302 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221ae68, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce0d80, 0xc0006161a0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ce0d80, {0xc0006161a0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ce0d80, {0xc0006161a0, 0xc000901f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f110, {0xc0006161a0, 0x46b213, 0x617f6333})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f110}, {0xc0006161a0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000584840, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 303 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000584840, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 584 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000472790)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 568 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 467 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007304d0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 468 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730580)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 305 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000584b00, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 399 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 400 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 1245991 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000fb65f0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1255008 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001788b40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbc6e0, {0x1f7b560, 0xc000cd5860}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xac)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016f900, 0xc0026cc440, 0xc0026cc410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016f900, {0x1f7b560, 0xc000cd5860})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00059def0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x20, {0xc000fe9800, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00059def0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00059def0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0xc001c74840, {0xc000fe9800, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00059def0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0xc00218ff30, 0xc00059df20, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00059def0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x8, {0xc000ed4b60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001fe3a40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x237ac70, 0xc00059de60, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001fe3a40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001fe3a40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc001fe3a40}, 0xc001640006, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc002663e00}, 0xc001fe39c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc002663e00}, 0xc001fe39c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002663b60, {0x2392298, 0xc002663e00})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002663da0, {0x2392298, 0xc002663e00}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001788240, {0x23922d0, 0xc00059c630}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00059c630}, 0xc001300a70, 0x1bd58c0, 0x1300a90)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc00059c330, 0xc001fe3140)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc000683020}, 0xc001fe3140)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000683020}, {0x1f353a0, 0xc001fe3140})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc000683020}, {0x1f353a0, 0xc001fe3140}, 0x237bc38, 0xc0020b3b90)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000683020}, {0x1f353a0, 0xc001fe3140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc000682e70, {0x23922d0, 0xc000682ea0}, {0x1f353a0, 0xc001fe3140}, 0x7f6bf2166268, 0xc001fe3180)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682ea0}, {0x1f353a0, 0xc001fe3140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000682ea0}, {0x1f353a0, 0xc001fe3140}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682ea0}, {0x1f353a0, 0xc001fe3140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140}, 0x40, 0xc001fe3180)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140}, 0xc000fe9620, 0xc0020b3b90)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000682e10}, 0xc0026638c0, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0005d2120, 0xc00070a240, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0005d2120, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 585 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000472840)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 297 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a1b8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce1780, 0xc000616030, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ce1780, {0xc000616030, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ce1780, {0xc000616030, 0xc000b71f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f0f8, {0xc000616030, 0x46b213, 0x617f6333})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f0f8}, {0xc000616030, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000700dc0, 0x5ba0e0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 298 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000700dc0, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 387 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 386 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0001c8fc0, {0x2392228, 0xc0007bf540})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1270886 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00083d440)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbc8f0, {0x1f7b7c0, 0xc001a11220}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8c00, 0xc001615bf8, 0xc001615bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8c00, {0x1f7b7c0, 0xc001a11220})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x20, {0xc001b58640, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0xc001bca580, {0xc001b58640, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0xc00218fb70, 0xc001346120, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x8, {0xc000756a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0013442c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x237ac70, 0xc001346060, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0013442c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0013442c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0013442c0}, 0xc000e28680, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc00133d9e0}, 0xc001344240, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc00133d9e0}, 0xc001344240, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001616480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001616528, {0x235a640, 0xc0006dc210}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf20dc980, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00133d9e0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001fbfbe0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001616778, {0x235a640, 0xc0000a9fb0}, 0x622d646363342d31)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc00133d9e0}, 0xc001327f40)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0018b2db8, 0x4c1cb4}, {0xc057f6a3664625cd, 0x6240989f9c}, {0xc0018b2db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc00133d9e0}, 0xc001327f40)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40}, 0xc001b58520, 0xc001eedb30)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40}, 0x15, 0xc001327f80)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40}, 0x2076951, 0xc001327f80)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40}, 0x237bc38, 0xc001327f80)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00133d890}, {0x1f33320, 0xc001327f40}, 0x1e5a940, 0xc001327f80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d890}, {0x1f33320, 0xc001327f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40}, 0x23c27b0, 0xc001327f80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002994b60, {0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40}, 0xc001b58520, 0xc001eedb30)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00133d860}, 0xc000efcae0, 0xc0005754c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001be1200, 0xc000cf2e10, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001be1200, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1242712 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001a70fc0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc0c60, {0x1f7b7c0, 0xc000dee8c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016fd00, 0xc00080a4d0, 0xc00080a4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016fd00, {0x1f7b7c0, 0xc000dee8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x20, {0xc0016f7560, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0xc001fb0dc0, {0xc0016f7560, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0xc001fb6920, 0xc001e91950, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x8, {0xc000ed4b60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001dfd780}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x237ac70, 0xc001e91890, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001dfd780}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001dfd780}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000097128, {0x2392228, 0xc001dfd780}, 0xc001640a56, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc00068e150, {0x2392298, 0xc001ee2ea0}, 0xc001dfd6c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc00062aa20, {0x2392298, 0xc001ee2ea0}, 0xc001dfd6c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001ee2b40, {0x2392298, 0xc001ee2ea0})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001ee2e40, {0x2392298, 0xc001ee2ea0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a70900, {0x23922d0, 0xc001e90d50}, 0x23)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0016409f0, {0x23922d0, 0xc001e90d50}, 0xc001dbb920, 0x1, 0x1dbb930)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc001e90cf0, 0xc001dfcdc0)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc001e90a50}, 0xc001dfcdc0)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e90a50}, {0x1f35540, 0xc001dfcdc0})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001e90a50}, {0x1f35540, 0xc001dfcdc0}, 0x237bc38, 0xc00237f080)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e90a50}, {0x1f35540, 0xc001dfcdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001e90900}, {0x1f35540, 0xc001dfcdc0}, 0xc001dfc840, 0xc001dfce40)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e90900}, {0x1f35540, 0xc001dfcdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e90900}, {0x1f35540, 0xc001dfcdc0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e90900}, {0x1f35540, 0xc001dfcdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0}, 0x40, 0xc001dfce40)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0}, 0x203000, 0x203000)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0}, 0xc0016f7400, 0xc00237f080)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001e908d0}, 0xc001ee2840, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001d89e60, 0xc00070a240, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001d89e60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 941 [select]:
go.temporal.io/server/service/history/shard.(*ControllerImpl).shardManagementPump(0xc00068a500)
/temporal/service/history/shard/controller_impl.go:312 +0x148
created by go.temporal.io/server/service/history/shard.(*ControllerImpl).Start
/temporal/service/history/shard/controller_impl.go:145 +0xaf
goroutine 1245920 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000886a20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001badd90, {0x1f7b560, 0xc000cd5f40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036400, 0xc001b84440, 0xc001b84410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036400, {0x1f7b560, 0xc000cd5f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002408750}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x20, {0xc0013e9420, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002408750}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002408750}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0xc000db6160, {0xc0013e9420, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002408750}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0xc001fb6920, 0xc002408780, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002408750}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x8, {0xc000ed4b60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000f27040}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x237ac70, 0xc0024086c0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000f27040}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000f27040}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc000f27040}, 0xc0016b6f36, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc0025afda0}, 0xc000f26fc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc0025afda0}, 0xc000f26fc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0025afb00, {0x2392298, 0xc0025afda0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0025afd40, {0x2392298, 0xc0025afda0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a61e60, {0x23922d0, 0xc001fd79e0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001fd79e0}, 0xc0011f58f0, 0x1bd58c0, 0x11f5900)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001fd7980, 0xc000f26740)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001fd76e0}, 0xc000f26740)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001fd76e0}, {0x1f353a0, 0xc000f26740})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001fd76e0}, {0x1f353a0, 0xc000f26740}, 0x237bc38, 0xc0022f8078)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd76e0}, {0x1f353a0, 0xc000f26740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001fd7560, {0x23922d0, 0xc001fd7590}, {0x1f353a0, 0xc000f26740}, 0xc001fd68a0, 0xc000f26780)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd7590}, {0x1f353a0, 0xc000f26740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001fd7590}, {0x1f353a0, 0xc000f26740}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd7590}, {0x1f353a0, 0xc000f26740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740}, 0x40, 0xc000f26780)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740}, 0xc0013e9300, 0xc0022f8078)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001fd7500}, 0xc0025af860, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001a61320, 0xc00070a240, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001a61320, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1270931 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0016d2540, {0x2392298, 0xc000ebe300}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0016c1680, {0x23922d0, 0xc0015ecf60}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001898090, {0x23922d0, 0xc0015ecf60}, 0xc0015d4b00, 0x1, 0x15d4b10)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0015ecf00, 0xc0015d6b00)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0015ecc90}, 0xc0015d6b00)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0015ecc90}, {0x1f35540, 0xc0015d6b00})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0015ecc90}, {0x1f35540, 0xc0015d6b00}, 0x237bc38, 0xc002423290)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecc90}, {0x1f35540, 0xc0015d6b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0015ecb40}, {0x1f35540, 0xc0015d6b00}, 0xc0015d6500, 0xc0015d6b80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecb40}, {0x1f35540, 0xc0015d6b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0015ecb40}, {0x1f35540, 0xc0015d6b00}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecb40}, {0x1f35540, 0xc0015d6b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00}, 0x40, 0xc0015d6b80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00}, 0xc001b59bc0, 0xc002423290)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0015ecb10}, 0xc000ebe2a0, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008a97a0, 0xc00070a240, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008a97a0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 499 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 498 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0006186c0, {0x2392228, 0xc000e7e640})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1068202 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc002371ea0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1255013 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0022c8ea0, {0x2392298, 0xc0023e83c0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0010fcd80, {0x23922d0, 0xc00070ae40}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00070ae40}, 0xc0007d8280, 0x1bd58c0, 0x7d8290)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc00070abd0, 0xc001fe3e40)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc00070a840}, 0xc001fe3e40)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00070a840}, {0x1f353a0, 0xc001fe3e40})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc00070a840}, {0x1f353a0, 0xc001fe3e40}, 0x237bc38, 0xc001f9e0f0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a840}, {0x1f353a0, 0xc001fe3e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00070a690}, {0x1f353a0, 0xc001fe3e40}, 0x203000, 0xc001fe3e80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a690}, {0x1f353a0, 0xc001fe3e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00070a690}, {0x1f353a0, 0xc001fe3e40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a690}, {0x1f353a0, 0xc001fe3e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40}, 0x40, 0xc001fe3e80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40}, 0xc000fe9ac0, 0xc001f9e0f0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00070a5a0}, 0xc0023e8360, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001788c60, 0xc00070a240, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001788c60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 775 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 763 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0001c8d80, {0x2392228, 0xc000ef6340})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 764 [select, 7 minutes]:
go.temporal.io/server/service/history/events.(*NotifierImpl).dequeueHistoryEventNotifications(0xc000680370)
/temporal/service/history/events/notifier.go:234 +0x105
created by go.temporal.io/server/service/history/events.(*NotifierImpl).Start
/temporal/service/history/events/notifier.go:253 +0x71
goroutine 1250119 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0016e4900)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001a0f6b0, {0x1f7b560, 0xc00068bb80}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9900, 0xc000d61d18, 0xc000d61ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9900, {0x1f7b560, 0xc00068bb80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x20, {0xc0018811a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0xc000ec8c60, {0xc0018811a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0xc00218fb70, 0xc0016d0060, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x8, {0xc000756a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000ffdf40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x237ac70, 0xc0016b3f80, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000ffdf40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000ffdf40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000ffdf40}, 0xc000ff86c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc0016b38c0}, 0xc000ffdec0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc0016b38c0}, 0xc000ffdec0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc000d625a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000d62648, {0x235a640, 0xc0006dc210}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d6cc8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc0016b38c0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00074b560})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000d62898, {0x235a640, 0xc0000a9fb0}, 0x392d643232342d61)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc0016b38c0}, 0xc0016b3710)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001bd0db8, 0x4c1cb4}, {0xc057f6a22685af7e, 0x6116d23755}, {0xc001bd0db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc0016b38c0}, 0xc0016b3710)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710}, 0xc0018810e0, 0xc001668108)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710}, 0x15, 0xc000ffdc80)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710}, 0x207690b, 0xc000ffdc80)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710}, 0x237bc38, 0xc000ffdc80)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0016b3770}, {0x1f33180, 0xc0016b3710}, 0x1e5a940, 0xc000ffdc80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b3770}, {0x1f33180, 0xc0016b3710})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710}, 0x9, 0xc000ffdc80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710}, 0xc0007f7768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710}, 0xc0018810e0, 0xc001668108)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0016b36b0}, 0xc00275c240, 0xc0005754c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00077f440, 0xc000cf2e10, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00077f440, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1059766 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0020c0960, {0x2392228, 0xc001053180})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1050 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc0010fd7a0)
/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1748 +0x1e65
goroutine 1064362 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc002620460)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1073651 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00205d4a0, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001da53e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45
goroutine 647 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0001c9200, {0x2392228, 0xc000ef6100})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 648 [select, 2 minutes]:
go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).purgeProcessor(0xc000124150)
/temporal/common/persistence/namespaceReplicationQueue.go:404 +0xed
created by go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).Start
/temporal/common/persistence/namespaceReplicationQueue.go:123 +0x71
goroutine 649 [select, 7 minutes]:
go.temporal.io/server/service/frontend.(*VersionChecker).versionCheckLoop(0xc0006a4d20)
/temporal/service/frontend/versionChecker.go:88 +0xe5
created by go.temporal.io/server/service/frontend.(*VersionChecker).Start.func1
/temporal/service/frontend/versionChecker.go:70 +0x5f
goroutine 1270897 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0017ce120, {0x2392298, 0xc000efd8c0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0017898c0, {0x23922d0, 0xc00136c9f0}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001f5fce0, {0x23922d0, 0xc00136c9f0}, 0xc001341b60, 0x1, 0x1341b70)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc00136c990, 0xc0013456c0)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc00136c720}, 0xc0013456c0)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00136c720}, {0x1f35540, 0xc0013456c0})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc00136c720}, {0x1f35540, 0xc0013456c0}, 0x237bc38, 0xc001f90c00)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c720}, {0x1f35540, 0xc0013456c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00136c5d0}, {0x1f35540, 0xc0013456c0}, 0xc001345040, 0xc001345740)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c5d0}, {0x1f35540, 0xc0013456c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00136c5d0}, {0x1f35540, 0xc0013456c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c5d0}, {0x1f35540, 0xc0013456c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0}, 0x40, 0xc001345740)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0}, 0xc001b58ba0, 0xc001f90c00)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00136c5a0}, 0xc000efd860, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000e5d8c0, 0xc00070a240, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000e5d8c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1245874 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008f3320)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1c30, {0x1f7b560, 0xc001769b80}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8100, 0xc00168e440, 0xc00168e410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8100, {0x1f7b560, 0xc001769b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x20, {0xc000fe9740, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0xc00176e2c0, {0xc000fe9740, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0xc001fb6920, 0xc001edaa80, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x8, {0xc000ed4b60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e5b080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x237ac70, 0xc001eda960, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e5b080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e5b080}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc000e5b080}, 0xc001cf6486, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc0025ae5a0}, 0xc000e5b000, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc0025ae5a0}, 0xc000e5b000, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002016180, {0x2392298, 0xc0025ae5a0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0020164e0, {0x2392298, 0xc0025ae5a0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005de360, {0x23922d0, 0xc001fff050}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001fff050}, 0xc00117ceb0, 0x1bd58c0, 0x117cec0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001ffeff0, 0xc000f12080)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001ffed50}, 0xc000f12080)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ffed50}, {0x1f353a0, 0xc000f12080})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001ffed50}, {0x1f353a0, 0xc000f12080}, 0x237bc38, 0xc0025942d0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffed50}, {0x1f353a0, 0xc000f12080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001ffebd0, {0x23922d0, 0xc001ffec00}, {0x1f353a0, 0xc000f12080}, 0x7f6bf2195f30, 0xc000f120c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffec00}, {0x1f353a0, 0xc000f12080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ffec00}, {0x1f353a0, 0xc000f12080}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffec00}, {0x1f353a0, 0xc000f12080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080}, 0x40, 0xc000f120c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080}, 0xc0013e8de0, 0xc0025942d0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001ffeb70}, 0xc0025ae540, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ef1440, 0xc00070a240, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ef1440, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1250270 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0025f5b60, {0x2392228, 0xc0027555c0})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1245994 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc000ba58c0, {0x2392228, 0xc000fca200})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1073579 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a470, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001f8b480, 0xc001ef2000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f8b480, {0xc001ef2000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001f8b480, {0xc001ef2000, 0x60100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f808, {0xc001ef2000, 0x466a4e, 0x4383a7})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002337e60, {0xc00038a040, 0x9, 0xc001a23d18})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc002337e60}, {0xc00038a040, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00038a040, 0x9, 0xc001a23d6f}, {0x23585c0, 0xc002337e60})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00038a000)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0005c0000)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1250269 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0025f5b60, {0x2392228, 0xc0027555c0})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1270927 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ffeb40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbcdc0, {0x1f7b7c0, 0xc000dee1e0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9b00, 0xc0010e1bf8, 0xc0010e1bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9b00, {0x1f7b7c0, 0xc000dee1e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x20, {0xc001b59760, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0xc0015c6000, {0xc001b59760, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0xc00218fb70, 0xc0015bf8f0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x8, {0xc000756a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0015d6140}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x33, 0xc0015bf830, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0015d6140}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0015d6140}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0015d6140}, 0xc000e29000, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc0015bf1a0}, 0xc0015d60c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc0015bf1a0}, 0xc0015d60c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0010e2480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0010e2528, {0x235a640, 0xc0006dc210}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2180980, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0015bf1a0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001e0cfd0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0010e2778, {0x235a640, 0xc0000a9fb0}, 0x612d666564342d63)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc0015bf1a0}, 0xc00137bdc0)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000bbcdb8, 0x4c1cb4}, {0xc057f6a366b64ed9, 0x624108c8a5}, {0xc000bbcdb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc0015bf1a0}, 0xc00137bdc0)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0}, 0xc001b59600, 0xc0024229a8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0}, 0x15, 0xc00137be00)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0}, 0x2076951, 0xc00137be00)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0}, 0x237bc38, 0xc00137be00)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0015bf050}, {0x1f33320, 0xc00137bdc0}, 0x1e5a940, 0xc00137be00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf050}, {0x1f33320, 0xc00137bdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0}, 0x23c27b0, 0xc00137be00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002994b60, {0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0}, 0x40, 0xc00005f800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0}, 0xc001b59600, 0xc0024229a8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0015bf020}, 0xc0017fdb00, 0xc0005754c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0010fd8c0, 0xc000cf2e10, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0010fd8c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1245896 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000846de0, {0x2392298, 0xc0021adb00}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001ba4000, {0x23922d0, 0xc001edb410}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001edb410}, 0xc0012e3c10, 0x1bd58c0, 0x12e3c20)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001edb200, 0xc000e5b280)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001edaf60}, 0xc000e5b280)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001edaf60}, {0x1f353a0, 0xc000e5b280})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001edaf60}, {0x1f353a0, 0xc000e5b280}, 0x237bc38, 0xc001cfe7f8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edaf60}, {0x1f353a0, 0xc000e5b280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001635710, {0x23922d0, 0xc001edae10}, {0x1f353a0, 0xc000e5b280}, 0xc001634a50, 0xc000e5b2c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edae10}, {0x1f353a0, 0xc000e5b280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001edae10}, {0x1f353a0, 0xc000e5b280}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edae10}, {0x1f353a0, 0xc000e5b280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280}, 0x40, 0xc000e5b2c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280}, 0xc000fe9880, 0xc001cfe7f8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001edadb0}, 0xc0021adaa0, 0xc00062aae0)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0008f3440, 0xc00070a240, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0008f3440, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1073675 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a640, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0020ac400, 0xc002226000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0020ac400, {0xc002226000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0020ac400, {0xc002226000, 0x60100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f960, {0xc002226000, 0x466a4e, 0x4383a7})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00256e240, {0xc00038a580, 0x9, 0xc00134ad18})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc00256e240}, {0xc00038a580, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00038a580, 0x9, 0xc00134ad6f}, {0x23585c0, 0xc00256e240})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00038a540)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0005c05a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1250144 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ec6ea0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001647550, {0x1f7b560, 0xc001a11900}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a72700, 0xc001b5dd18, 0xc001b5dce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a72700, {0x1f7b560, 0xc001a11900})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x20, {0xc0016f66e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0xc0019d3340, {0xc0016f66e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0xc00218fb70, 0xc0018f5f20, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x8, {0xc000756a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001ca3c40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x237ac70, 0xc0018f5e60, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001ca3c40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001ca3c40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001ca3c40}, 0xc00249e420, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc0018f55c0}, 0xc001ca3bc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc0018f55c0}, 0xc001ca3bc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001b5e5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001b5e648, {0x235a640, 0xc0006dc210}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2123e90, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc0018f55c0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00218ef30})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001b5e898, {0x235a640, 0xc0000a9fb0}, 0x612d643330342d31)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc0018f55c0}, 0xc0018f5410)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0018dedb8, 0x4c1cb4}, {0xc057f6a22709e97a, 0x6117567147}, {0xc0018dedb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc0018f55c0}, 0xc0018f5410)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410}, 0xc0016f6580, 0xc001866e40)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410}, 0x15, 0xc001ca3980)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410}, 0x207690b, 0xc001ca3980)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410}, 0x237bc38, 0xc001ca3980)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0018f5470}, {0x1f33180, 0xc0018f5410}, 0x1e5a940, 0xc001ca3980)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f5470}, {0x1f33180, 0xc0018f5410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410}, 0x9, 0xc001ca3980)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410}, 0xc001a1e768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001a1eb60, {0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410}, 0x40, 0xc00005e400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410}, 0xc0016f6580, 0xc001866e40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0018f53e0}, 0xc00260d3e0, 0xc0005754c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008e8360, 0xc000cf2e10, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008e8360, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1245787 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000484a20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1810, {0x1f7b560, 0xc001768d20}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c9d00, 0xc0016dfd18, 0xc0016dfce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c9d00, {0x1f7b560, 0xc001768d20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00189be90}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x20, {0xc000fe8d00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00189be90}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00189be90}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0xc000eee580, {0xc000fe8d00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00189be90}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0xc00218fb70, 0xc00189bec0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00189be90}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x8, {0xc000756a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e3c7c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x237ac70, 0xc00189be00, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e3c7c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e3c7c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000e3c7c0}, 0xc001a6ec30, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc00189b4a0}, 0xc000e3c740, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc00189b4a0}, 0xc000e3c740, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0016e05a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016e0648, {0x235a640, 0xc0006dc210}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21c9820, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc00189b4a0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001742280})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016e0898, {0x235a640, 0xc0000a9fb0}, 0x612d386233342d65)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc00189b4a0}, 0xc00189b2f0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001b44db8, 0x4c1cb4}, {0xc057f6a1e74c1ce2, 0x60dbfddaba}, {0xc001b44db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc00189b4a0}, 0xc00189b2f0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0}, 0xc000fe8c40, 0xc0017393c8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0}, 0x15, 0xc000e3c500)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0}, 0x207690b, 0xc000e3c500)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0}, 0x237bc38, 0xc000e3c500)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00189b350}, {0x1f33180, 0xc00189b2f0}, 0x1e5a940, 0xc000e3c500)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b350}, {0x1f33180, 0xc00189b2f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0}, 0x9, 0xc000e3c500)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0}, 0xc002239768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002239b60, {0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0}, 0x40, 0xc00005e400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0}, 0xc000fe8c40, 0xc0017393c8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00189b2c0}, 0xc002167800, 0xc0005754c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008a90e0, 0xc000cf2e10, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008a90e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1073676 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002095540, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00256e2a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45
goroutine 1254951 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0007de0a0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1270992 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00077b560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ee8c60, {0x1f7b7c0, 0xc000cd4e60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000b8100, 0xc00080dbf8, 0xc00080dbc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000b8100, {0x1f7b7c0, 0xc000cd4e60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x20, {0xc001cb43a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0xc0019d2160, {0xc001cb43a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0xc00218fb70, 0xc0013eef30, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x8, {0xc000756a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0013e6ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x237ac70, 0xc0013eee70, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0013e6ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0013e6ec0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0013e6ec0}, 0xc000f00fc0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc0013ee810}, 0xc0013e6e40, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc0013ee810}, 0xc0013e6e40, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00080e480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc00080e528, {0x235a640, 0xc0006dc210}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf211ea10, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0013ee810}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001dee530})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc00080e778, {0x235a640, 0xc0000a9fb0}, 0x612d306465342d30)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc0013ee810}, 0xc0013e6b40)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000bbedb8, 0x4c1cb4}, {0xc057f6a3680f8aca, 0x62426204a1}, {0xc000bbedb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc0013ee810}, 0xc0013e6b40)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40}, 0xc001cb41c0, 0xc001ebcd38)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40})