single liveview mp3 transcription example app

speech_to_text_example.exs

Application.put_env(:sample, PhoenixDemo.Endpoint,
  http: [ip: {127, 0, 0, 1}, port: 8080],
  server: true,
  live_view: [signing_salt: "bumblebee"],
  secret_key_base: String.duplicate("b", 64),
  pubsub_server: PhoenixDemo.PubSub
)

Mix.install([
  {:plug_cowboy, "~> 2.6"},
  {:jason, "~> 1.4"},
  {:phoenix, "~> 1.7.0"},
  {:phoenix_live_view, "~> 0.18.18"},
  {:mp3_duration, "~> 0.1.0"},
  # Bumblebee and friends
  {:bumblebee, "~> 0.3.0"},
  {:nx, "~> 0.5.1"},
  {:exla, "~> 0.5.1"}
])

Application.put_env(:nx, :default_backend, EXLA.Backend)

defmodule PhoenixDemo.Layouts do
  use Phoenix.Component

  def render("live.html", assigns) do
    ~H"""
    <script type="module">
      import AudioRecorder from 'https://cdn.jsdelivr.net/npm/audio-recorder-polyfill/index.js'
      import mpegEncoder from 'https://cdn.jsdelivr.net/npm/audio-recorder-polyfill/mpeg-encoder/index.js'
      AudioRecorder.encoder = mpegEncoder
      AudioRecorder.prototype.mimeType = 'audio/mpeg'
      window.MediaRecorder = AudioRecorder
    </script>
    <script src="https://cdn.jsdelivr.net/npm/phoenix@1.7.0-rc.0/priv/static/phoenix.min.js">
    </script>
    <script
      src="https://cdn.jsdelivr.net/npm/phoenix_live_view@0.18.3/priv/static/phoenix_live_view.min.js"
    >
    </script>
    <script>
      let Hooks = {};

      Hooks.Demo = {
        mounted() {
          let mediaRecorder;
          const audioChunks = [];

          this.handleEvent("stop", () => {
            mediaRecorder.stop();
          });

          this.handleEvent("start", () => {
            navigator.mediaDevices.getUserMedia({ audio: true })
              .then(stream => {
                mediaRecorder = new MediaRecorder(stream);

                mediaRecorder.addEventListener("dataavailable", event => {
                  audioChunks.push(event.data);
                });

                mediaRecorder.addEventListener("stop", () => {
                  const audioBlob = new Blob(audioChunks);
                  this.upload("audio", [audioBlob]);
                });

                mediaRecorder.start();
              });
          });
        },
      };

      const liveSocket = new window.LiveView.LiveSocket("/live", window.Phoenix.Socket, { hooks: Hooks });
      liveSocket.connect();
    </script>
    <script src="https://cdn.tailwindcss.com">
    </script>
    <%= @inner_content %>
    """
  end
end

defmodule PhoenixDemo.ErrorView do
  def render(_, _), do: "error"
end

defmodule PhoenixDemo.SampleLive do
  use Phoenix.LiveView, layout: {PhoenixDemo.Layouts, :live}

  @impl true
  def mount(_, _, socket) do
    socket =
      socket
      |> assign(audio: nil, recording: false, task: nil)
      |> allow_upload(:audio, accept: :any, progress: &handle_progress/3, auto_upload: true)
      |> stream(:segments, [], dom_id: &"ss-#{&1.ss}")

    {:ok, socket}
  end

  @impl true
  def handle_event("start", _value, socket) do
    socket = socket |> push_event("start", %{})
    {:noreply, assign(socket, recording: true)}
  end

  @impl true
  def handle_event("stop", _value, %{assigns: %{recording: recording}} = socket) do
    socket = if recording, do: socket |> push_event("stop", %{}), else: socket
    {:noreply, assign(socket, recording: false)}
  end

  @impl true
  def handle_event("noop", %{}, socket) do
    # We need phx-change and phx-submit on the form for live uploads
    {:noreply, socket}
  end

  @impl true
  def handle_info({ref, results}, socket) when socket.assigns.task.ref == ref do
    socket = socket |> assign(task: nil)

    socket =
      results
      |> Enum.reduce(socket, fn {_duration, ss, text}, socket ->
        socket |> stream_insert(:segments, %{ss: ss, text: text})
      end)

    {:noreply, socket}
  end

  @impl true
  def handle_info(_, socket) do
    {:noreply, socket}
  end

  def handle_progress(:audio, entry, socket) when entry.done? do
    path =
      consume_uploaded_entry(socket, entry, fn upload ->
        dest = Path.join(["priv", "static", "uploads", Path.basename(upload.path)])
        File.cp!(upload.path, dest)
        {:ok, dest}
      end)

    {:ok, %{duration: duration}} = Mp3Duration.parse(path)

    task =
      speech_to_text(duration, path, 20, fn ss, text ->
        {duration, ss, text}
      end)

    {:noreply, assign(socket, task: task)}
  end

  def handle_progress(_name, _entry, socket), do: {:noreply, socket}

  def speech_to_text(duration, path, chunk_time, func) do
    Task.async(fn ->
      format = get_format()

      0..duration//chunk_time
      |> Task.async_stream(
        fn ss ->
          args = ~w(-ac 1 -ar 16k -f #{format} -ss #{ss} -t #{chunk_time} -v quiet -)
          {data, 0} = System.cmd("ffmpeg", ["-i", path] ++ args)
          {ss, Nx.Serving.batched_run(PhoenixDemo.Serving, Nx.from_binary(data, :f32))}
        end,
        max_concurrency: 4,
        timeout: :infinity
      )
      |> Enum.map(fn {:ok, {ss, %{results: [%{text: text}]}}} ->
        func.(ss, text)
      end)
    end)
  end

  def get_format() do
    case System.endianness() do
      :little -> "f32le"
      :big -> "f32be"
    end
  end

  @impl true
  def render(assigns) do
    ~H"""
    <div class="h-screen">
      <div id="transcript" phx-update="stream" class="pt-4">
        <div
          :for={{id, segment} <- @streams.segments}
          id={id}
          class="flex w-full justify-center items-center text-blue-400 font-bold"
        >
          <%= segment.text %>
        </div>
      </div>
      <div class="flex h-screen w-full justify-center items-center">
        <form phx-change="noop" phx-submit="noop" class="hidden">
          <.live_file_input upload={@uploads.audio} />
        </form>
        <div id="mic-element" class="flex h-20 w-20 rounded-full bg-gray-700 p-2" phx-hook="Demo">
          <div
            :if={@task}
            class="h-full w-full bg-white rounded-full ring-2 ring-white animate-spin border-4 border-solid border-blue-500 border-t-transparent"
          >
          </div>
          <button
            :if={!@task && !@recording}
            class="h-full w-full bg-red-500 rounded-full ring-2 ring-white"
            type="button"
            phx-click="start"
            class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded"
          >
          </button>
          <button
            :if={!@task && @recording}
            class="h-full w-full bg-red-500 rounded-full ring-2 ring-white animate-pulse"
            type="button"
            phx-click="stop"
            class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded"
          >
          </button>
        </div>
      </div>
    </div>
    """
  end
end

defmodule PhoenixDemo.Router do
  use Phoenix.Router

  import Phoenix.LiveView.Router

  pipeline :browser do
    plug(:accepts, ["html"])
  end

  scope "/", PhoenixDemo do
    pipe_through(:browser)

    live("/", SampleLive, :index)
  end
end

defmodule PhoenixDemo.Endpoint do
  use Phoenix.Endpoint, otp_app: :sample

  socket("/live", Phoenix.LiveView.Socket)
  plug(PhoenixDemo.Router)
end

# Application startup
Nx.default_backend(EXLA.Backend)

{:ok, whisper} = Bumblebee.load_model({:hf, "openai/whisper-tiny"})
{:ok, featurizer} = Bumblebee.load_featurizer({:hf, "openai/whisper-tiny"})
{:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, "openai/whisper-tiny"})
{:ok, generation_config} = Bumblebee.load_generation_config({:hf, "openai/whisper-tiny"})

serving =
  Bumblebee.Audio.speech_to_text(whisper, featurizer, tokenizer, generation_config,
    defn_options: [compiler: EXLA]
  )

{:ok, _} =
  Supervisor.start_link(
    [
      {Phoenix.PubSub, name: PhoenixDemo.PubSub},
      {Nx.Serving, serving: serving, name: PhoenixDemo.Serving, batch_timeout: 100},
      PhoenixDemo.Endpoint
    ],
    strategy: :one_for_one
  )

path = Path.join(["priv", "static", "uploads"])
File.mkdir_p(path)

Process.sleep(:infinity)