PyZeebe and FastAPI hanging thread issue

.gitignore

.venv

README.md

Setup

python -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
# alternatively, also install pyzeebe from path:
pip install -e $PATH_TO_PYZEEBE_PARENT/pyzeebe

Run / reproduce

# in one terminal
python main.py

# in the other
curl localhost:5001/

Output

[2020-11-03 13:25:40,665] [81833 - MainThread] [DEBUG] asyncio - Using selector: KqueueSelector
[2020-11-03 13:25:40,694] [81833 - MainThread] [INFO] uvicorn.error - Started server process [81833]
[2020-11-03 13:25:40,694] [81833 - MainThread] [INFO] uvicorn.error - Waiting for application startup.
[2020-11-03 13:25:40,695] [81833 - MainThread] [INFO] uvicorn.error - Application startup complete.
[2020-11-03 13:25:40,696] [81833 - MainThread] [INFO] uvicorn.error - Uvicorn running on http://0.0.0.0:5001 (Press CTRL+C to quit)
[2020-11-03 13:25:49,359] [81833 - Thread-2] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.IDLE
[2020-11-03 13:25:49,359] [81833 - Thread-2] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Connected to localhost:26599
[2020-11-03 13:25:49,359] [81833 - Thread-2] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.CONNECTING
[2020-11-03 13:25:49,360] [81833 - Thread-2] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Connecting to localhost:26599.
[2020-11-03 13:25:49,361] [81833 - Thread-2] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.TRANSIENT_FAILURE
[2020-11-03 13:25:49,361] [81833 - Thread-2] [WARNING] pyzeebe.grpc_internals.zeebe_adapter_base - Lost connection to localhost:26599. Retrying...
[2020-11-03 13:25:49,361] [81833 - MainThread] [INFO] uvicorn.access - 127.0.0.1:63982 - "GET / HTTP/1.1" 500
[2020-11-03 13:25:49,362] [81833 - MainThread] [ERROR] uvicorn.error - Exception in ASGI application
Traceback (most recent call last):
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/uvicorn/protocols/http/h11_impl.py", line 389, in run_asgi
    result = await app(self.scope, self.receive, self.send)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__
    return await self.app(scope, receive, send)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/uvicorn/middleware/message_logger.py", line 65, in __call__
    raise exc from None
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/uvicorn/middleware/message_logger.py", line 61, in __call__
    await self.app(scope, inner_receive, inner_send)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/fastapi/applications.py", line 179, in __call__
    await super().__call__(scope, receive, send)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/starlette/applications.py", line 111, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/starlette/middleware/errors.py", line 181, in __call__
    raise exc from None
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/starlette/middleware/errors.py", line 159, in __call__
    await self.app(scope, receive, _send)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/starlette/exceptions.py", line 82, in __call__
    raise exc from None
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/starlette/exceptions.py", line 71, in __call__
    await self.app(scope, receive, sender)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/starlette/routing.py", line 566, in __call__
    await route.handle(scope, receive, send)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/starlette/routing.py", line 227, in handle
    await self.app(scope, receive, send)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/starlette/routing.py", line 41, in app
    response = await func(request)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/fastapi/routing.py", line 183, in app
    dependant=dependant, values=values, is_coroutine=is_coroutine
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/fastapi/routing.py", line 135, in run_endpoint_function
    return await run_in_threadpool(dependant.call, **values)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/starlette/concurrency.py", line 34, in run_in_threadpool
    return await loop.run_in_executor(None, func, *args)
  File "/Users/kristofferb/.pyenv/versions/3.7.6/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "main.py", line 14, in read_root
    queued = client.run_workflow("test-workflow", {})
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/pyzeebe/client/client.py", line 37, in run_workflow
    version=version)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/pyzeebe/grpc_internals/zeebe_workflow_adapter.py", line 22, in create_workflow_instance
    self._create_workflow_errors(rpc_error, bpmn_process_id, version, variables)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/pyzeebe/grpc_internals/zeebe_workflow_adapter.py", line 46, in _create_workflow_errors
    self._common_zeebe_grpc_errors(rpc_error)
  File "/Users/kristofferb/Code/temp/uvicorn-zeeebe-issue/.venv/lib/python3.7/site-packages/pyzeebe/grpc_internals/zeebe_adapter_base.py", line 71, in _common_zeebe_grpc_errors
    raise ZeebeGatewayUnavailable()
pyzeebe.exceptions.zeebe_exceptions.ZeebeGatewayUnavailable
[2020-11-03 13:25:50,361] [81833 - Thread-3] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.CONNECTING
[2020-11-03 13:25:50,361] [81833 - Thread-3] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Connecting to localhost:26599.
[2020-11-03 13:25:50,362] [81833 - Thread-4] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.TRANSIENT_FAILURE
[2020-11-03 13:25:50,362] [81833 - Thread-4] [WARNING] pyzeebe.grpc_internals.zeebe_adapter_base - Lost connection to localhost:26599. Retrying...
[2020-11-03 13:25:51,941] [81833 - Thread-5] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.CONNECTING
[2020-11-03 13:25:51,941] [81833 - Thread-5] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Connecting to localhost:26599.
[2020-11-03 13:25:52,103] [81833 - Thread-6] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.TRANSIENT_FAILURE
[2020-11-03 13:25:52,104] [81833 - Thread-6] [WARNING] pyzeebe.grpc_internals.zeebe_adapter_base - Lost connection to localhost:26599. Retrying...
[2020-11-03 13:25:54,929] [81833 - Thread-7] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.CONNECTING
[2020-11-03 13:25:54,929] [81833 - Thread-7] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Connecting to localhost:26599.
[2020-11-03 13:25:54,929] [81833 - Thread-8] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.TRANSIENT_FAILURE
[2020-11-03 13:25:54,929] [81833 - Thread-8] [WARNING] pyzeebe.grpc_internals.zeebe_adapter_base - Lost connection to localhost:26599. Retrying...
[2020-11-03 13:25:58,449] [81833 - Thread-9] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.CONNECTING
[2020-11-03 13:25:58,450] [81833 - Thread-9] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Connecting to localhost:26599.
[2020-11-03 13:25:58,947] [81833 - Thread-10] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.TRANSIENT_FAILURE
[2020-11-03 13:25:58,947] [81833 - Thread-10] [WARNING] pyzeebe.grpc_internals.zeebe_adapter_base - Lost connection to localhost:26599. Retrying...
[2020-11-03 13:26:05,869] [81833 - Thread-11] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.CONNECTING
[2020-11-03 13:26:05,869] [81833 - Thread-11] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Connecting to localhost:26599.
[2020-11-03 13:26:06,533] [81833 - Thread-12] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.TRANSIENT_FAILURE
[2020-11-03 13:26:06,533] [81833 - Thread-12] [WARNING] pyzeebe.grpc_internals.zeebe_adapter_base - Lost connection to localhost:26599. Retrying...
[2020-11-03 13:26:16,208] [81833 - Thread-13] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.CONNECTING
[2020-11-03 13:26:16,208] [81833 - Thread-13] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Connecting to localhost:26599.
[2020-11-03 13:26:17,207] [81833 - Thread-14] [DEBUG] pyzeebe.grpc_internals.zeebe_adapter_base - Grpc channel connectivity changed to: ChannelConnectivity.TRANSIENT_FAILURE
[2020-11-03 13:26:17,208] [81833 - Thread-14] [WARNING] pyzeebe.grpc_internals.zeebe_adapter_base - Lost connection to localhost:26599. Retrying...
^C[2020-11-03 13:26:24,086] [81833 - MainThread] [INFO] uvicorn.error - Shutting down
[2020-11-03 13:26:24,189] [81833 - MainThread] [INFO] uvicorn.error - Waiting for application shutdown.
[2020-11-03 13:26:24,190] [81833 - MainThread] [INFO] uvicorn.error - Application shutdown complete.
[2020-11-03 13:26:24,190] [81833 - MainThread] [INFO] uvicorn.error - Finished server process [81833]

main.py

import logging

import pyzeebe
import uvicorn
from fastapi import FastAPI

app = FastAPI()

NON_LISTENING_PORT = 26599

@app.get("/")
def read_root():
    client = pyzeebe.ZeebeClient(hostname="localhost", port=NON_LISTENING_PORT)
    queued = client.run_workflow("test-workflow", {})

    return {"queued": queued}


if __name__ == "__main__":
    log_format =  "[%(asctime)s] [%(process)d - %(threadName)s] [%(levelname)s] %(name)s - %(message)s"
    logging.basicConfig(format=log_format, level=logging.DEBUG)
    uvicorn.run(app, host="0.0.0.0", port=5001, log_config=None)

requirements.txt

certifi==2020.6.20
chardet==3.0.4
click==7.1.2
fastapi==0.61.1
grpcio==1.32.0
h11==0.11.0
idna==2.10
oauthlib==3.1.0
protobuf==3.13.0
pydantic==1.7.2
pyzeebe==2.2.2
requests==2.24.0
requests-oauthlib==1.3.0
six==1.15.0
starlette==0.13.6
typing-extensions==3.7.4.3
urllib3==1.25.11
uvicorn==0.12.2
zeebe-grpc==0.24.3.3