Skip to content

Instantly share code, notes, and snippets.

@sergeyprokudin

sergeyprokudin/llff_poses_to_pytorch3d_camera.ipynb

Last active May 24, 2023 00:00
Show Gist options
  • Save sergeyprokudin/a27ad1d43922eafa5f1926359ced222e to your computer and use it in GitHub Desktop.
Save sergeyprokudin/a27ad1d43922eafa5f1926359ced222e to your computer and use it in GitHub Desktop.
Download ZIP
llff_poses_to_pytorch3d_camera.ipynb
Display the source blob
Display the rendered blob
Raw
llff_poses_to_pytorch3d_camera.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "llff_poses_to_pytorch3d_camera.ipynb",
"provenance": [],
"collapsed_sections": [],
"authorship_tag": "ABX9TyMyjz0uGdMV9GYZ/oRD7OmW",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"accelerator": "GPU",
"gpuClass": "standard"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/sergeyprokudin/a27ad1d43922eafa5f1926359ced222e/llff_poses_to_pytorch3d_camera.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"source": [
"# This is a small code snippet investigating the way to convert LLFF dataset \n",
"# format (pose_bounds.npy) to Pytorch3d camera data \n",
"#\n",
"# Sergey Prokudin (sergey.prokudin@gmail.com)\n",
"# Zürich, August, 2022\n",
"\n",
"\n",
"# Some useful links utilised during the investigation:\n",
"\n",
"# LLFF conversion:\n",
"# https://github.com/Fyusion/LLFF/blob/master/llff/poses/pose_utils.py\n",
"\n",
"# \"OpenCV camera to PyTorch3D PerspectiveCameras\"\n",
"# https://github.com/facebookresearch/pytorch3d/issues/522\n",
"\n",
"# \"How to convert LLFF camera poses to Pytorch3D cameras? \"\n",
"# https://github.com/facebookresearch/pytorch3d/issues/1125\n",
"\n",
"# \"Great work! How to convert original camera pose from NeRF to Pytorch3d style?\"\n",
"# https://github.com/facebookresearch/pytorch3d/issues/883\n",
"\n",
"# NeRF Fern LLFF data:\n",
"# https://drive.google.com/drive/folders/1McCFABesPQxXDimpudJHd7xw-0IlWSba?usp=sharing\n",
"\n",
"# NeRF Fern data pytorch3d format:\n",
"# https://dl.fbaipublicfiles.com/pytorch3d_nerf_data/fern.pth\n",
"\n",
"# LLFF pose format description\n",
"# https://github.com/Fyusion/LLFF"
],
"metadata": {
"id": "Aj8Zt9ubqsye"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# install pytorch3d\n",
"\n",
"import os\n",
"import sys\n",
"import torch\n",
"need_pytorch3d=False\n",
"try:\n",
" import pytorch3d\n",
"except ModuleNotFoundError:\n",
" need_pytorch3d=True\n",
"if need_pytorch3d:\n",
" if torch.__version__.startswith(\"1.10.\") and sys.platform.startswith(\"linux\"):\n",
" # We try to install PyTorch3D via a released wheel.\n",
" pyt_version_str=torch.__version__.split(\"+\")[0].replace(\".\", \"\")\n",
" version_str=\"\".join([\n",
" f\"py3{sys.version_info.minor}_cu\",\n",
" torch.version.cuda.replace(\".\",\"\"),\n",
" f\"_pyt{pyt_version_str}\"\n",
" ])\n",
" !pip install fvcore iopath\n",
" !pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html\n",
" else:\n",
" # We try to install PyTorch3D from source.\n",
" !curl -LO https://github.com/NVIDIA/cub/archive/1.10.0.tar.gz\n",
" !tar xzf 1.10.0.tar.gz\n",
" os.environ[\"CUB_HOME\"] = os.getcwd() + \"/cub-1.10.0\"\n",
" !pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'\n",
"\n",
"# also clone the pytorch3d repo for some auxiliary functions\n",
"!git clone https://github.com/facebookresearch/pytorch3d/"
],
"metadata": {
"id": "1WpOczawV62g"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import torch\n",
"import sys\n",
"import numpy as np\n",
"from pytorch3d.renderer import PerspectiveCameras\n",
"\n",
"sys.path.insert(0, '/content/pytorch3d/pytorch3d')\n",
"from implicitron.dataset.load_llff import load_llff_data\n",
"\n",
"\n",
"# Copyright (c) Meta Platforms, Inc. and affiliates.\n",
"# All rights reserved.\n",
"#\n",
"# This source code is licensed under the BSD-style license found in the\n",
"# LICENSE file in the root directory of this source tree.\n",
"\n",
"\n",
"# the function is taken from:\n",
"# https://github.com/facebookresearch/pytorch3d/blob/bd93e9ce21a0b73f1097181e6e6c0bc32e71b7ef/pytorch3d/implicitron/dataset/single_sequence_dataset.py#L163\n",
"\n",
"def _interpret_blender_cameras(\n",
" poses: torch.Tensor, focal: float):\n",
" \"\"\"\n",
" Convert 4x4 matrices representing cameras in blender format\n",
" to PyTorch3D format.\n",
"\n",
" Args:\n",
" poses: N x 3 x 4 camera matrices\n",
" focal: ndc space focal length\n",
" \"\"\"\n",
" pose_target_cameras = []\n",
" for pose_target in poses:\n",
" pose_target = pose_target[:3, :4]\n",
" mtx = torch.eye(4, dtype=pose_target.dtype)\n",
" mtx[:3, :3] = pose_target[:3, :3].t()\n",
" mtx[3, :3] = pose_target[:, 3]\n",
" mtx = mtx.inverse()\n",
"\n",
" # flip the XZ coordinates.\n",
" mtx[:, [0, 2]] *= -1.0\n",
"\n",
" Rpt3, Tpt3 = mtx[:, :3].split([3, 1], dim=0)\n",
"\n",
" focal_length_pt3 = torch.FloatTensor([[focal, focal]])\n",
" principal_point_pt3 = torch.FloatTensor([[0.0, 0.0]])\n",
"\n",
" cameras = PerspectiveCameras(\n",
" focal_length=focal_length_pt3,\n",
" principal_point=principal_point_pt3,\n",
" R=Rpt3[None],\n",
" T=Tpt3,\n",
" )\n",
" pose_target_cameras.append(cameras)\n",
" return pose_target_cameras\n",
"\n",
"# A functional simplified version of the following data loader:\n",
"# https://github.com/facebookresearch/pytorch3d/blob/a0f786f4cf600a8b1741ef8b5b24bc99272ab0e7/pytorch3d/implicitron/dataset/load_llff.py#L307\n",
"\n",
"def load_llff_data_pytorch3d(basedir, factor=None):\n",
"\n",
" images, poses, bds = load_llff_data(basedir, factor=factor)\n",
"\n",
" hwf = poses[0, :3, -1]\n",
" poses = poses[:, :3, :4]\n",
"\n",
" i_test = np.arange(images.shape[0])[::8]\n",
" i_test_index = set(i_test.tolist())\n",
" i_train = np.array(\n",
" [i for i in np.arange(images.shape[0]) if i not in i_test_index]\n",
" )\n",
" i_split = (i_train, i_test, i_test)\n",
" H, W, focal = hwf\n",
" focal_ndc = 2 * focal / min(H, W)\n",
" images = torch.from_numpy(images)#.permute(0, 3, 1, 2)\n",
" poses = torch.from_numpy(poses)\n",
"\n",
" poses = _interpret_blender_cameras(poses, focal_ndc)\n",
"\n",
" return images, poses, bds"
],
"metadata": {
"id": "nYNcWK1qMbmL"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# download the data for comparison: original NeRF fern data \n",
"\n",
"%cd /content\n",
"!gdown 1pQzsS29cJjgoum9kJQo95rJfTObOlp1M\n",
"!unzip /content/fern_nerf_original.zip"
],
"metadata": {
"id": "MnCSEUDLovXF"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# use our loader\n",
"images, poses, bds = load_llff_data_pytorch3d('/content/fern', factor=None)"
],
"metadata": {
"id": "h4j27hUooZpB"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# now let's load the already preprocessed data from Meta and compare the results\n",
"# (with poses already converted to the correct camera mats):\n",
"\n",
"!wget https://dl.fbaipublicfiles.com/pytorch3d_nerf_data/fern.pth\n",
"\n",
"data_pytorch3d = torch.load('/content/fern.pth')"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "7wAK4vCcLEnT",
"outputId": "63240cbe-6e74-4c5a-d379-e503d1474e9e"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"--2022-08-06 10:09:20-- https://dl.fbaipublicfiles.com/pytorch3d_nerf_data/fern.pth\n",
"Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.74.142, 172.67.9.4, 104.22.75.142, ...\n",
"Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.74.142|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 4021 (3.9K) [application/zip]\n",
"Saving to: ‘fern.pth’\n",
"\n",
"fern.pth 100%[===================>] 3.93K --.-KB/s in 0s \n",
"\n",
"2022-08-06 10:09:21 (67.6 MB/s) - ‘fern.pth’ saved [4021/4021]\n",
"\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"print(\"our code rotation matrix:\\n %s\" % poses[0].R)\n",
"\n",
"print(\"pytorch3d rotation matrix:\\n %s\" % data_pytorch3d['cameras']['R'][0])"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "VXEbpSzupSNn",
"outputId": "9484b621-74e2-4ba0-eff5-3eb7354a5e87"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"converted rotation matrix:\n",
" tensor([[[-0.9957, -0.0208, 0.0903],\n",
" [-0.0250, 0.9986, -0.0460],\n",
" [-0.0892, -0.0481, -0.9948]]])\n",
"pytorch3d rotation matrix:\n",
" tensor([[-0.9957, -0.0208, 0.0903],\n",
" [-0.0250, 0.9986, -0.0460],\n",
" [-0.0892, -0.0481, -0.9948]])\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"print(\"our code translation vector:\\n %s\" % poses[0].T)\n",
"\n",
"print(\"pytorch3d translation vector:\\n %s\" % data_pytorch3d['cameras']['T'][0])"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ROWXvFgKp3Wg",
"outputId": "271ce7df-5613-451a-822d-d9c0cd2133cd"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"our code rotation matrix:\n",
" tensor([[-0.2998, -0.1390, 0.0737]])\n",
"pytorch3d rotation matrix:\n",
" tensor([-0.2998, -0.1390, 0.0737])\n"
]
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment