A particular sentry + finch setup that results in sentry entering a possibly infinite error loop with some possible workarounds.

error_loop.exs

Mix.install([:sentry, :finch, :jason])

defmodule Sentry.FinchClient do
  @moduledoc false
  # adapts https://github.com/getsentry/sentry-elixir/blob/master/lib/sentry/hackney_client.ex
  @behaviour Sentry.HTTPClient

  @finch_name S.Finch

  @impl true
  def child_spec do
    child_spec = {Finch, name: @finch_name, pools: %{default: [size: 1, count: 1]}}
    Supervisor.child_spec(child_spec, [])
  end

  @impl true
  def post(url, headers, body) do
    req = Finch.build(:post, url, headers, body)

    case Finch.request(req, @finch_name, receive_timeout: 5000) do
      {:ok, %Finch.Response{status: status, body: body, headers: headers}} ->
        {:ok, status, headers, body}

      {:error, _reason} = failure ->
        failure
    end
  end
end

Application.put_all_env(
  sentry: [
    client: Sentry.FinchClient,
    environment_name: :test,
    included_environments: [:test],
    dsn: System.fetch_env!("SENTRY_DSN")
  ]
)

Application.put_env(:logger, Sentry.LoggerBackend, capture_log_messages: true)

Logger.add_backend(Sentry.LoggerBackend)
:ok = Application.ensure_started(:sentry)

Enum.each(1..1000, fn i -> Sentry.capture_message("event #{i}") end)

receive do
  :never -> :ok
end

Once Sentry client starts sending enough events, nimble_pool which finch uses for managing http1 connections starts timing out and exiting and finch relays the exit to the caller which makes Sentry task to terminate and generate another exception thus entering an error loop generating more exceptions.

The exeption looks like this

12:12:12.866 [error] Task #PID<0.391.0> started from #PID<0.288.0> terminating
** (stop) exited in: NimblePool.checkout(#PID<0.347.0>)
    ** (EXIT) time out
    (finch 0.12.0) lib/finch/http1/pool.ex:57: Finch.HTTP1.Pool.request/5
    (finch 0.12.0) lib/finch.ex:313: anonymous fn/4 in Finch.request/3
    (telemetry 1.1.0) /Users/ruslan/Developer/sentry-finch-issue/deps/telemetry/src/telemetry.erl:320: :telemetry.span/3
    (s 0.1.0) lib/finch_client.ex:18: Sentry.FinchClient.post/3
    (sentry 8.0.6) lib/sentry/client.ex:210: Sentry.Client.request/3
    (sentry 8.0.6) lib/sentry/client.ex:192: Sentry.Client.try_request/5
    (sentry 8.0.6) lib/sentry/client.ex:167: anonymous fn/4 in Sentry.Client.do_send_event/3
    (elixir 1.13.4) lib/task/supervised.ex:89: Task.Supervised.invoke_mfa/2
Function: #Function<2.43295134/0 in Sentry.Client.do_send_event/3>
    Args: []

To reproduce:

#!/usr/bin/env bash
export SENTRY_DSN=https://1a77dc7a687646b28c1b1b6cee1ef4a9@o1012425.ingest.sentry.io/6601933
curl -LO https://gist.githubusercontent.com/ruslandoga/9bb8c0a27f8ffd7661d34b8957c9d9ac/raw/c0eb3a448c0493f08bef38286a92436c56333308/error_loop.exs
elixir error_loop.exs

Possible fix #1

Catch the exit thus preventing the sentry task crash and instead make it go into the retry branch.

defmodule Sentry.FinchClient do
  @moduledoc false
  # adapts https://github.com/getsentry/sentry-elixir/blob/master/lib/sentry/hackney_client.ex
  @behaviour Sentry.HTTPClient

  @finch_name S.Finch

  @impl true
  def child_spec do
    child_spec = {Finch, name: @finch_name, pools: %{default: [size: 1, count: 1]}}
    Supervisor.child_spec(child_spec, [])
  end

  @impl true
  def post(url, headers, body) do
    req = Finch.build(:post, url, headers, body)

    case Finch.request(req, @finch_name, receive_timeout: 5000) do
      {:ok, %Finch.Response{status: status, body: body, headers: headers}} ->
        {:ok, status, headers, body}

      {:error, _reason} = failure ->
        failure
    end
+ catch
+   :exit, {:timeout, {NimblePool, :checkout, [_pid]}} ->
+    {:error, :timeout}
  end
end

Possible fix #2

Use http2 protocol for which finch doesn't use nimble_pool and returns mint's error tuples in a similar scenario instead of raising:

 {:error, %Mint.HTTPError{module: Mint.HTTP2, reason: :too_many_concurrent_requests}}

Which prevents sentry task from crashing and instead makes it retry the request.

defmodule Sentry.FinchClient do
  @moduledoc false
  # adapts https://github.com/getsentry/sentry-elixir/blob/master/lib/sentry/hackney_client.ex
  @behaviour Sentry.HTTPClient

  @finch_name S.Finch

  @impl true
  def child_spec do
-   child_spec = {Finch, name: @finch_name, pools: %{default: [size: 1, count: 1]}}
+   child_spec =
+     {Finch, name: @finch_name, pools: %{default: [protocol: :http2, size: 1, count: 1]}}

    Supervisor.child_spec(child_spec, [])
  end

  @impl true
  def post(url, headers, body) do
    req = Finch.build(:post, url, headers, body)

    case Finch.request(req, @finch_name, receive_timeout: 5000) do
      {:ok, %Finch.Response{status: status, body: body, headers: headers}} ->
        {:ok, status, headers, body}

      {:error, _reason} = failure ->
        failure
    end
  end
end

Possible fix #3

Sentry package could add try/catch around calling the custom client. Pseudocode

Current

# Somewhere in sentry package

if custom_client do
  custom_client.post(url, headers, body) # If this throws, things go bad
end

Proposed

# Somewhere in sentry package

if custom_client do
  try do
    custom_client.post(url, headers, body)
  catch
    e -> {:error, e}
  end
end

This way, no matter what the client does, the error loop is prevented. We should ask - is there any reason for the Sentry package to report an exception that happened in the custom client?

I don't think so, and I think the error should be suppressed. If the client throws, it means sentry cannot report the error. It seems absurd to me to report an error that happened during reporting an error.