Render 3DDFA.ipynb

{
 "cells": [
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "%matplotlib qt\nimport matplotlib.pyplot as plt\nimport cv2\nimport numpy as np\nimport menpo.io as mio\nimport menpo3d.io as m3dio\n\nfrom pathlib import Path\nfrom scipy.io import loadmat\n\nfrom menpo.image import MaskedImage, Image\nfrom menpo.shape import PointCloud, TriMesh\nfrom menpo.transform import (Translation, AlignmentSimilarity, Scale, \n                             Homogeneous, Rotation, ThinPlateSplines)\nfrom menpo.visualize import print_progress\n\nfrom menpo3d.unwrap import optimal_cylindrical_unwrap\nfrom menpo3d.rasterize import GLRasterizer, model_to_clip_transform, clip_to_image_transform\nfrom menpofit.builder import align_shapes\nfrom menpo.model import PCAModel\nfrom menpowidgets import visualize_shape_model, visualize_pointclouds\n\nfrom rendering import (convert_3ddfa_to_original, cylindrical_snapping, \n                       retrieve_camera_matrix, yaw_rotation, pitch_rotation, \n                       roll_rotation)\n\nnp.set_printoptions(precision=3, suppress=True)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "trimmed_mask = mio.import_pickle('~/Dropbox/phd/3ddfa_to_trimmed_no_neck_mask.pkl.gz')\noriginal_template = mio.import_pickle('~/Dropbox/phd/mein3d_fw_correspond_mean.pkl.gz')\nmein3d_identity_model = mio.import_pickle('~/Dropbox/phd/mein3d_fw_correspond_trimmed_no_neck_all_200.pkl.gz')\n\ntddfa_expression_model = mio.import_pickle('~/Dropbox/phd/3ddfa_fw_trimmed_no_neck_29.pkl.gz')",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "template = mein3d_identity_model.mean()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Real Image"
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "DATASETS_PATH = Path('/vol/atlas/databases')\nTDDFA_PATH = DATASETS_PATH / '3ddfa'",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": true,
    "trusted": true
   },
   "cell_type": "code",
   "source": "ibug_image_paths = list(filter(lambda x: x.stem != \"image_092_01\", mio.image_paths('/vol/atlas/databases/ibug')))\nafw_image_paths = list(mio.image_paths('/vol/atlas/databases/afw'))\nlfpw_image_paths = list(mio.image_paths('/vol/atlas/databases/lfpw/**/*'))",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "def get_3ddfa_data(path):\n    folder = path.parent.name\n    main_folder = folder.upper()\n    stem = path.stem\n    if folder in {'testset', 'trainset'}:\n        folder = Path(*path.parts[-3:-1])\n        main_folder = folder.parent.name.upper()\n    if main_folder == 'LFPW':\n        stem = 'image_{}_{}'.format(folder.name[:-3].lower(), \n                                    path.stem.split('_')[-1])\n    \n    image_original = mio.import_image(DATASETS_PATH / '{}/{}'.format(folder, path.name))\n    image_3ddfa = mio.import_image(TDDFA_PATH / '300W-3D/{}/{}.jpg'.format(main_folder, stem))\n\n    fitted_mesh = loadmat(str(TDDFA_PATH / '300W-3D-Face/{}/{}.mat'.format(main_folder, stem)))['Fitted_Face']\n    fit_info = loadmat(str(TDDFA_PATH / '300W-3D/{}/{}.mat'.format(main_folder, stem)))\n    print(fit_info['Pose_Para'])\n    \n    fitted_3d_mesh, fitted_2d_mesh =  convert_3ddfa_to_original(fitted_mesh, fit_info, \n                                                                image_3ddfa, image_original,\n                                                                original_template.trilist)\n    return fitted_3d_mesh, fitted_2d_mesh, image_original",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "fitted_3d_mesh, fitted_2d_mesh, image_original = get_3ddfa_data(ibug_image_paths[87])\n# Rotate to upright to remove roll\nupright_theta = np.arctan2(*AlignmentSimilarity(fitted_3d_mesh, \n                                                original_template).h_matrix[:2, 0][::-1])\nimage_original, roll_remove_tr = image_original.rotate_ccw_about_centre(upright_theta, \n                                                                        degrees=False, \n                                                                        return_transform=True)\nfitted_2d_mesh = roll_remove_tr.pseudoinverse().apply(fitted_2d_mesh)\n\nview_t, c_t, proj_t = retrieve_camera_matrix(image_original, fitted_2d_mesh, fitted_3d_mesh)\nif fitted_3d_mesh.n_points != np.count_nonzero(trimmed_mask):\n    fitted_2d_mesh = fitted_2d_mesh.from_mask(trimmed_mask)\n    fitted_3d_mesh = fitted_3d_mesh.from_mask(trimmed_mask)\n\nrasterizer = GLRasterizer(height=image_original.height, \n                          width=image_original.width, \n                          view_matrix=view_t.h_matrix,\n                          projection_matrix=proj_t.h_matrix)\nprint(view_t.compose_before(proj_t).apply(fitted_3d_mesh).bounds())\nreal_instance_image = rasterizer.rasterize_mesh_with_shape_image(fitted_3d_mesh)[1]\n\n# projected_fitted_3d_mesh = fitted_3d_mesh.project_barycentric_coordinates(*ts_to_ut_bary)\n# projected_fitted_3d_mesh = TriMesh(projected_fitted_3d_mesh.points, \n#                                    trilist=template_scale.trilist)\n# projected_template_in_image = rasterizer.model_to_image_transform.apply(projected_fitted_3d_mesh)\n\n# instance = cylindrical_snapping(real_instance_image, \n#                                 projected_template_in_image, \n#                                 projected_fitted_3d_mesh, \n#                                 u_t_shape_image, u_t_shape_index_image, \n#                                 verbose=False)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# %matplotlib inline\nimage_original.view()\nfitted_2d_mesh.view(marker_size=1)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# %matplotlib inline\n# image_original.view()\n# projected_template_in_image.view(marker_size=1)",
   "execution_count": null,
   "outputs": []
  }
 ],
 "metadata": {
  "language_info": {
   "file_extension": ".py",
   "name": "python",
   "version": "3.5.1",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "mimetype": "text/x-python"
  },
  "widgets": {
   "version": "1.1.2",
   "state": {}
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}