Fill Missing DateTime and GPS Fields from Google Photos Takeout JSON (Elixir Livebook)

fill_missing_datetime_and_gps_fields_from_google_photos_takeout_json.livemd

Fill Missing DateTime and GPS Fields from Google Photos Takeout JSON

Section

Mix.install([
  {:kino, "~> 0.5.0"},
  {:exiftool, "~> 0.2.0"},
  {:timex, "~> 3.7"},
  {:flow, "~> 1.2"}
])

EXIF Update Module

defmodule EXIFUpdate do
  alias EXIFUpdate.Utils

  def update_file(filename, dir) do
    filepath = Path.join(dir, filename)
    # IO.puts("Working on: #{filepath}")

    split_filename_on_parantheses = String.split(filename, "(")

    json_filename =
      if Kernel.length(split_filename_on_parantheses) == 1 do
        Path.join(dir, [split_filename_on_parantheses, ".json"])
      else
        name = Enum.at(split_filename_on_parantheses, 0)
        multiple_with_ext = Enum.at(split_filename_on_parantheses, 1) |> String.split(")")
        multiple = multiple_with_ext |> Enum.at(0)
        extension = multiple_with_ext |> Enum.at(1)

        Path.join(dir, [name, extension, "(", multiple, ")", ".json"])
      end

    {:ok, data} = Exiftool.execute([json_filename])

    %{
      "photo_taken_time_timestamp" => json_photo_taken_time_timestamp,
      "creation_time_timestamp" => json_creation_time_timestamp,
      "geo_data_exif_altitude" => json_geo_data_exif_altitude,
      "geo_data_exif_latitude" => json_geo_data_exif_latitude,
      "geo_data_exif_longitude" => json_geo_data_exif_longitude
    } = data

    create_date = Utils.json_timestamp_to_exif_datetime(json_creation_time_timestamp)
    date_time_original = Utils.json_timestamp_to_exif_datetime(json_photo_taken_time_timestamp)
    gps_altitude = json_geo_data_exif_altitude |> String.to_float()
    gps_latitude = json_geo_data_exif_latitude |> String.to_float()
    gps_longitude = json_geo_data_exif_longitude |> String.to_float()

    exif_fields =
      Utils.build_exif_fields(
        filepath,
        create_date,
        date_time_original,
        gps_altitude,
        gps_latitude,
        gps_longitude
      )

    # |> IO.inspect(label: "EXIF Field Changes")

    # IO.puts("Writing: #{filepath}")
    Utils.write_exif_data(filepath, exif_fields)
    # IO.puts("")
  end
end

Utils Module

defmodule EXIFUpdate.Utils do
  def build_exif_fields(
        filepath,
        create_date,
        date_time_original,
        gps_altitude,
        gps_latitude,
        gps_longitude
      ) do
    {:ok, current_exif_fields} = Exiftool.execute([filepath])

    # IO.inspect(current_exif_fields)

    is_png? = String.slice(filepath, -4..-1) == ".png"

    %{}
    |> __MODULE__.maybe_add_date_time_fields(
      current_exif_fields,
      is_png?,
      create_date,
      date_time_original
    )
    |> __MODULE__.maybe_add_gps_fields(
      current_exif_fields,
      gps_latitude,
      gps_longitude,
      gps_altitude
    )
  end

  @doc """
  Add new CreateDate or DateTimeOriginal only if they don't already exist.
  """
  def maybe_add_date_time_fields(
        new_exif_fields,
        current_exif_fields,
        is_png?,
        create_date,
        date_time_original
      ) do
    Enum.reduce(
      [
        {"create_date", "CreateDate", create_date},
        {"date_time_original", "DateTimeOriginal", date_time_original}
      ],
      new_exif_fields,
      fn {exif_key, write_key, val}, acc ->
        if Map.has_key?(current_exif_fields, exif_key) do
          acc
        else
          if is_png? == true and write_key == "DateTimeOriginal" do
            # DateCreated needs to be written to EXIF ModifyDate 
            # as "YYYY-MM-DD hh:mm:ss" for Google Photos to properly store
            # PNG files at the correct date/time.
            modify_date =
              val
              |> Timex.parse!("{ISO:Extended}")
              |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")

            acc
            |> Map.put(write_key, val)
            |> Map.put("DateCreated", modify_date)
          else
            Map.put(acc, write_key, val)
          end
        end
      end
    )
  end

  @doc """
  Add new GPSLatitude and GPSLongitude if:
   - none of these fields currently exist: GPSPosition, GPSLatitude, and GPSLongitude and
   - either the new GPSLatitude or the GPSLongitude does not equal 0.0.
  ExifTool will automatically figure out GPSLatitudeRef/GPSLongitudeRef from the same GPSLatitude/GPSLongitude value.
  Add new GPSAltitude:
   - only if new GPSLatitude/GPSLongitude are being added and
   - if new GPSAltitude input does not equal 0.0.
  """
  def maybe_add_gps_fields(
        new_exif_fields,
        current_exif_fields,
        gps_latitude,
        gps_longitude,
        gps_altitude
      ) do
    if should_add_gps_fields?(current_exif_fields) == true and
         (gps_latitude != 0.0 or gps_longitude != 0.0) do
      new_exif_fields
      |> (&Enum.reduce(
            [
              {"GPSLatitude", gps_latitude},
              {"GPSLatitudeRef", gps_latitude},
              {"GPSLongitude", gps_longitude},
              {"GPSLongitudeRef", gps_longitude}
            ],
            &1,
            fn {key, val}, acc ->
              Map.put(acc, key, val)
            end
          )).()
      |> maybe_add_gps_altitude(gps_altitude)
    else
      new_exif_fields
    end
  end

  defp should_add_gps_fields?(current_exif_fields) do
    Map.has_key?(current_exif_fields, "gps_position") == false and
      Map.has_key?(current_exif_fields, "gps_latitude") == false and
      Map.has_key?(current_exif_fields, "gps_longitude") == false
  end

  defp maybe_add_gps_altitude(new_exif_fields, gps_altitude) do
    if gps_altitude != 0.0 do
      new_exif_fields
      |> Map.put("GPSAltitude", gps_altitude)
      |> Map.put("GPSAltitudeRef", 0)
    else
      new_exif_fields
    end
  end

  def json_timestamp_to_exif_datetime(timestamp) do
    timestamp
    |> String.to_integer()
    |> Timex.from_unix()
    |> Timex.format!("{ISO:Extended}")
  end

  def write_exif_data(filepath, exif_fields) do
    exif_fields_params =
      Enum.reduce(exif_fields, [], fn
        {key, val}, acc -> ["-xmp:#{key}=\"#{val}\"" | acc]
      end)

    # IO.inspect(exif_fields_params)
    Exiftool.execute(exif_fields_params ++ ["-overwrite_original_in_place", filepath])
  end
end

Update Files

dir = "/home/user/Downloads/photos takeout/Photos from 2022/"
files = File.ls!(dir)

not_json_and_edited_filelist =
  files
  |> Enum.filter(fn filename ->
    filename |> String.contains?([".json", "-edited"]) == false
  end)

# Linear
# {usecs, :ok} =
#   :timer.tc(fn ->
#     for filename <- not_json_and_edited_filelist do
#       EXIFUpdate.update_file(filename, dir)
#     end

#     :ok
#   end)

# Batched Async
# {usecs, :ok} =
#   :timer.tc(fn ->
#     batches =
#       not_json_and_edited_filelist
#       |> Enum.chunk_every(400)
#       |> Enum.with_index(fn batch, i -> {i, batch} end)

#     for {i, batch} <- batches do
#       IO.puts("Batch #{i + 1} running")

#       tasks =
#         Enum.reduce(batch, [], fn filename, acc ->
#           [Task.async(fn -> EXIFUpdate.update_file(filename, dir) end) | acc]
#         end)

#       Task.await_many(tasks, :infinity)
#     end

#     :ok
#   end)

# Async Stream
# {usecs, :ok} =
#   :timer.tc(fn ->
#     stream =
#       Task.async_stream(
#         not_json_and_edited_filelist,
#         fn filename -> EXIFUpdate.update_file(filename, dir) end,
#         max_concurrency: 400,
#         timeout: :infinity,
#         ordered: false
#       )

#     for s <- stream do
#       s
#     end

#     # Enum.reduce(stream, 0, fn _, acc -> acc end)
#     :ok
#   end)

# Flow
{usecs, :ok} =
  :timer.tc(fn ->
    not_json_and_edited_filelist
    |> Flow.from_enumerable(max_demand: :erlang.system_info(:schedulers_online))
    |> Flow.map(&EXIFUpdate.update_file(&1, dir))
    |> Flow.partition()
    |> Flow.run()

    :ok
  end)

IO.inspect(usecs / 1_000_000)