imageortext2mesh.ipynb

image2pointcloud.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/kevincolten/56529dcb4a42ea3e159e76322a9980e6/image2pointcloud.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/openai/point-e.git trimesh ipdb"
      ],
      "metadata": {
        "id": "IiLp-_QhYEAR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dpcBCsv-X_fS"
      },
      "outputs": [],
      "source": [
        "from PIL import Image\n",
        "import torch\n",
        "import trimesh\n",
        "import skimage.measure\n",
        "from tqdm.auto import tqdm\n",
        "import matplotlib.pyplot as plt\n",
        "import ipdb\n",
        "\n",
        "from point_e.diffusion.configs import DIFFUSION_CONFIGS, diffusion_from_config\n",
        "from point_e.diffusion.sampler import PointCloudSampler\n",
        "from point_e.models.download import load_checkpoint\n",
        "from point_e.models.configs import MODEL_CONFIGS, model_from_config\n",
        "from point_e.util.plotting import plot_point_cloud\n",
        "from point_e.util.pc_to_mesh import marching_cubes_mesh\n",
        "from point_e.util.point_cloud import PointCloud\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "img1 = Image.open('table-1.jpeg')\n",
        "img2 = Image.open('table-2.jpeg')\n",
        "img3 = Image.open('table-3.jpeg')\n",
        "img4 = Image.open('table-4.jpeg')\n",
        "\n",
        "images = [\n",
        "    img1,\n",
        "    img2,\n",
        "    img3,\n",
        "    img4\n",
        "]\n",
        "\n",
        "prompt = 'brown table'"
      ],
      "metadata": {
        "id": "hzMAhgHwPB93"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "id60HymtX_fT"
      },
      "outputs": [],
      "source": [
        "print('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "print('creating base model...')\n",
        "\n",
        "# use base40M base300M or base1B for better image results\n",
        "base_name = 'base1B' if len(images) > 0 else 'base40M-textvec'\n",
        "base_model = model_from_config(MODEL_CONFIGS[base_name], device)\n",
        "base_model.eval()\n",
        "base_diffusion = diffusion_from_config(DIFFUSION_CONFIGS[base_name])\n",
        "\n",
        "print('creating upsample model...')\n",
        "upsampler_model = model_from_config(MODEL_CONFIGS['upsample'], device)\n",
        "upsampler_model.eval()\n",
        "upsampler_diffusion = diffusion_from_config(DIFFUSION_CONFIGS['upsample'])\n",
        "\n",
        "print('downloading base checkpoint...')\n",
        "base_model.load_state_dict(load_checkpoint(base_name, device))\n",
        "\n",
        "print('downloading upsampler checkpoint...')\n",
        "upsampler_model.load_state_dict(load_checkpoint('upsample', device))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VH_q9UncX_fU"
      },
      "outputs": [],
      "source": [
        "sampler = PointCloudSampler(\n",
        "    device=device,\n",
        "    models=[base_model, upsampler_model],\n",
        "    diffusions=[base_diffusion, upsampler_diffusion],\n",
        "    num_points=[1024, 4096 - 1024],\n",
        "    aux_channels=['R', 'G', 'B'],\n",
        "    guidance_scale=[3.0, 3.0],\n",
        "    model_kwargs_key_filter=('images' if len(images) else 'texts', ''),\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1vlPcQx5X_fU"
      },
      "outputs": [],
      "source": [
        "# Produce a sample from the model.\n",
        "samples = None\n",
        "for x in tqdm(sampler.sample_batch_progressive(\n",
        "    batch_size=(len(images) if len(images) > 0 else 1),\n",
        "    model_kwargs=dict(\n",
        "        images=images,\n",
        "        texts=[prompt]\n",
        "    )\n",
        ")):\n",
        "  samples = x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bP2P8RFZX_fU"
      },
      "outputs": [],
      "source": [
        "pc = sampler.output_to_point_clouds(samples)[0]\n",
        "pc.save('pointcloud.npz')\n",
        "fig = plot_point_cloud(pc, grid_size=3, fixed_bounds=((-0.75, -0.75, -0.75),(0.75, 0.75, 0.75)))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('creating SDF model...')\n",
        "name = 'sdf'\n",
        "model = model_from_config(MODEL_CONFIGS[name], device)\n",
        "model.eval()\n",
        "\n",
        "print('loading SDF model...')\n",
        "model.load_state_dict(load_checkpoint(name, device))"
      ],
      "metadata": {
        "id": "h-OpbNZKPrQu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load a point cloud we want to convert into a mesh.\n",
        "pc = PointCloud.load('pointcloud.npz')\n",
        "\n",
        "# Plot the point cloud as a sanity check.\n",
        "fig = plot_point_cloud(pc, grid_size=2)"
      ],
      "metadata": {
        "id": "-NjdMcj6QLHR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Produce a mesh (with vertex colors)\n",
        "mesh = marching_cubes_mesh(\n",
        "    pc=pc,\n",
        "    model=model,\n",
        "    batch_size=4096,\n",
        "    grid_size=128, # increase to 128 for resolution used in evals\n",
        "    progress=True,\n",
        ")"
      ],
      "metadata": {
        "id": "7TsYQu7AQMFE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Write the mesh to a PLY file to import into some other program.\n",
        "with open('mesh.ply', 'wb') as f:\n",
        "    mesh.write_ply(f)\n",
        "mesh = trimesh.load('mesh.ply')\n",
        "mesh.export('mesh.glb')\n",
        "mesh.export('mesh.obj')\n",
        "print('done')"
      ],
      "metadata": {
        "id": "PuwmVC-ZQRHk"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.9.9 64-bit ('3.9.9')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.9"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "b270b0f43bc427bcab7703c037711644cc480aac7c1cc8d2940cfaf0b447ee2e"
      }
    },
    "colab": {
      "provenance": [],
      "name": "imageortext2mesh.ipynb",
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "gpuClass": "premium"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}

Greetings, is there any advantage in using multiple images?

Thank you for providing this code.
I have the same question as Asdaois: what is the benefit of using multiple images?
Also, do you have any idea about generating an accurate 3D point cloud by using several images with different angles of the same object, using Point-E?

That was the idea, to get a more accurate point cloud with several images at different angles, but I don't think I saw much of a difference. This is just an example of how the API accepts more than one image, it might only use the first one inside somewhere.