ecto_log_to_sql_converter.exs

defmodule SqlParamsParser do
  import NimbleParsec

  string_ =
    ignore(string("\""))
    |> choice([utf8_string([not: ?"], min: 1), empty()])
    |> ignore(string("\""))
    |> reduce(:single_quote)

  decimal_ =
    ignore(string("#Decimal<"))
    |> ascii_string([not: ?>], min: 1)
    |> ignore(string(">"))

  integer_ =
    ascii_string([?0..?9], min: 1)

  date_time_ =
    ignore(string("#DateTime<"))
    |> ascii_string([not: ?>], min: 1)
    |> ignore(string(">"))

  naive_date_time_ =
    ignore(string("~N["))
    |> ascii_string([not: ?]], min: 1)
    |> ignore(string("]"))

  uuid_ =
    ignore(string("<<"))
    |> times(integer(max: 3) |> ignore(string(", ")), 15)
    |> integer(max: 3)
    |> ignore(string(">>"))
    |> reduce({:binary, :list_to_bin, []})
    |> map({UUID, :binary_to_string!, []})
    |> reduce(:single_quote)

  # Elixir Map Parsing (json/jsonb fields)

  ex_num_ =
    ascii_string([?0..?9, ?.], min: 1)
    |> map(:to_num!)

  # this assumes valid data, but is faster (less string munging) and
  # shorter
  def to_num!(s) do
    case Integer.parse(s) do
      {i, ""} -> i

      {_, "." <> _} ->
        {f, ""} = Float.parse(s)
        f
    end
  end

  ex_string_ =
    ignore(string(~S(")))
    |> choice([
      utf8_string([not: ?"], min: 1),
      empty() |> replace("")
    ])
    |> ignore(string(~S(")))


  ex_value_ =
    choice([
      ex_num_,
      ex_string_,
      decimal_,
      date_time_,
      naive_date_time_,
      string("nil") |> replace(nil),
      parsec(:ex_list_),
      parsec(:ex_map_)
    ])

  defcombinatorp :ex_list_,
    ignore(string("["))
    |> concat(ex_value_)
    |> repeat(ignore(string(", ")) |> concat(ex_value_))
    |> ignore(string("]"))

  # assumes atom keys
  ex_key_ =
    ascii_string([not: ?:], min: 1)
    |> map({String, :to_atom, []})

  # assumes atom keys
  ex_kv_pair_ =
    ex_key_
    |> ignore(string(": "))
    |> concat(ex_value_)
    |> reduce({List, :to_tuple, []})

  defcombinatorp :ex_map_,
    ignore(string("%{"))
    |> optional(
      ex_kv_pair_
      |> optional(repeat(
        ignore(string(", "))
        |> concat(ex_kv_pair_)
      ))
    )
    |> ignore(string("}"))
    |> reduce({Map, :new, []})

  map_ =
    parsec(:ex_map_)
    |> map({Jason, :encode!, []})
    |> reduce(:single_quote)



  defcombinatorp :param_,
    choice([
      date_time_ |> reduce(:single_quote),
      string_,
      decimal_,
      integer_,
      naive_date_time_ |> reduce(:single_quote),
      uuid_,
      string("nil") |> replace("NULL"),
      map_,
    ])

  params_ =
    parsec(:param_)
    |> repeat(ignore(string(", ")) |> parsec(:param_))

  def single_quote(s) do
    "'#{s}'"
  end

  defparsec :parse, params_
end

# assumes only Ecto SQL output is in the logs.
[
  "#{__DIR__}/this.log",
  "#{__DIR__}/that.log",
]
|> Enum.map(fn file ->

  File.stream!(file, [:utf8])
  |> Stream.map(fn s ->
    try do
      sql = s |> String.replace(~r{ \[.+$}, ";")
      param_str = s |> String.replace(~r{^.+?\[}, "") |> String.replace(~r{\][^\]]+$}, "")

      param_str
      |> SqlParamsParser.parse()
      |> case do
        {:ok, params, "", _, _, _} -> params
      end
      |> Enum.with_index(1)
      |> Enum.reverse() # reverse expand, so that $10 expands before $1
      |> Enum.reduce(sql, fn {param, i}, sql ->
        String.replace(sql, "$#{i}", "#{param}")
      end)
      |> String.replace("RETURNING", "ON CONFLICT DO NOTHING RETURNING")
    rescue
      e ->
        IO.inspect([e, s], label: "failure/input")
        System.stop(1)
    end
  end)
  |> Stream.into(File.stream!("#{file}.stage2", [:utf8]))
  |> Stream.run()
end)