Occluding Contour Snapping.ipynb

{
 "cells": [
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "%matplotlib qt\nimport matplotlib.pyplot as plt\nimport cv2\nimport numpy as np\nimport menpo.io as mio\nimport menpo3d.io as m3dio\n\nfrom pathlib import Path\nfrom scipy.io import loadmat\n\nfrom menpo.image import MaskedImage, Image\nfrom menpo.shape import PointCloud, TriMesh\nfrom menpo.transform import (Translation, AlignmentSimilarity, Scale, \n                             Homogeneous, Rotation, ThinPlateSplines)\nfrom menpo.visualize import print_progress\n\nfrom menpo3d.unwrap import optimal_cylindrical_unwrap\nfrom menpo3d.rasterize import GLRasterizer, model_to_clip_transform, clip_to_image_transform\nfrom menpofit.builder import align_shapes\nfrom menpo.model import PCAModel\nfrom menpowidgets import visualize_shape_model, visualize_pointclouds\n\nfrom rendering import (convert_3ddfa_to_original, cylindrical_snapping, \n                       retrieve_camera_matrix, yaw_rotation, pitch_rotation, \n                       roll_rotation)\n\nnp.set_printoptions(precision=3, suppress=True)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "trimmed_mask = mio.import_pickle('~/Dropbox/phd/3ddfa_to_trimmed_no_neck_mask.pkl.gz')\noriginal_template = mio.import_pickle('~/Dropbox/phd/mein3d_fw_correspond_mean.pkl.gz')\nmein3d_identity_model = mio.import_pickle('~/Dropbox/phd/mein3d_fw_correspond_trimmed_no_neck_all_200.pkl.gz')\n\ntddfa_expression_model = mio.import_pickle('~/Dropbox/phd/3ddfa_fw_trimmed_no_neck_29.pkl.gz')",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "print('Mein3D: ', mein3d_identity_model.n_features // 3)\nprint('3DDFA Expression: ', tddfa_expression_model.n_features // 3)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "template = mein3d_identity_model.mean()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "%matplotlib qt\ntemplate.view_landmarks()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "o_c_unwrap = optimal_cylindrical_unwrap(template)\nunwrapped_template = o_c_unwrap.apply(template)\n\nu_t_aspect_ratio = np.divide(*unwrapped_template.range()[:2])",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "%matplotlib inline\nu_t_clip_transform = model_to_clip_transform(unwrapped_template)\nu_t_clip = u_t_clip_transform.apply(unwrapped_template)\n\n# 75, 100, 150\nu_t_im_width = 150\nr = GLRasterizer(width=u_t_im_width, height=u_t_im_width / u_t_aspect_ratio)\n\nu_t_shape_image = r.rasterize_mesh_with_f3v_interpolant(u_t_clip, per_vertex_f3v=template.points)[1]\nu_t_shape_index_image = u_t_shape_image.from_vector(np.arange(u_t_shape_image.n_true_pixels()), n_channels=1)\nu_t_shape_image.view(channels=2)\nu_t_shape_index_image.view(new_figure=True)\nr._opengl.stop()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "u_t_shape_image_trilist = TriMesh.init_2d_grid(u_t_shape_image.shape).from_mask(\n                              u_t_shape_image.mask.mask.ravel()).trilist\ntemplate_scale = TriMesh(u_t_shape_image.as_vector(keep_channels=True).T, \n                         trilist=u_t_shape_image_trilist)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "ts_to_ut_bary = template.barycentric_coordinates_of_pointcloud(template_scale)\nprint(ts_to_ut_bary[1].shape)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "ut_to_ts_bary = template_scale.barycentric_coordinates_of_pointcloud(template)\nprint(ut_to_ts_bary[1].shape)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "mio.export_pickle({'unwrapped_to_template': ut_to_ts_bary, \n                   'template_to_unwrapped': ts_to_ut_bary,\n                   'template': template_scale}, \n                   '~/Dropbox/phd/unwrapped_to_render_template_150.pkl.gz',\n                   overwrite=True)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Model Image"
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "def generate_sensible_model_instance(template_im_width=300, verbose=False, \n                                     min_exp_ind=0, max_exp_ind=None, \n                                     n_simultaneous_exp=3,\n                                     min_exp_intensity=-0.25, max_exp_intensity=0.25):\n    if max_exp_ind is None:\n        max_exp_ind = tddfa_expression_model.n_components\n    \n    id_components = np.random.uniform(low=-2, high=2, size=mein3d_identity_model.n_components)\n    exp_components = np.zeros(tddfa_expression_model.n_components)\n    exp_ind = np.random.randint(low=min_exp_ind,\n                                high=max_exp_ind, \n                                size=n_simultaneous_exp)\n    exp_components[exp_ind] = np.random.uniform(low=min_exp_intensity, \n                                                high=max_exp_intensity, \n                                                size=n_simultaneous_exp)\n\n    id_instance = mein3d_identity_model.instance(id_components, \n                                                 normalized_weights=True)\n    exp_instance = tddfa_expression_model.instance(exp_components,\n                                                   normalized_weights=True)\n    model_exp_instance = TriMesh(id_instance.points + exp_instance.points, \n                                 trilist=id_instance.trilist)\n    \n    rand_yaw_theta = np.random.randint(low=-50, high=51)\n    rand_pitch_theta = np.random.randint(low=-40, high=41)\n    rotate_model = yaw_rotation(rand_yaw_theta)\n    rotate_model = rotate_model.compose_before(pitch_rotation(rand_pitch_theta))\n    if verbose:\n        print('Yaw: {}, Pitch: {}, Exp: {} at {:.3f}'.format(rand_yaw_theta, rand_pitch_theta, \n                                                      exp_ind, exp_components[exp_ind]))\n\n    m2c_template_tr = model_to_clip_transform(rotate_model.apply(model_exp_instance))\n    exp_template_scale = model_exp_instance.project_barycentric_coordinates(*ts_to_ut_bary)\n    exp_template_scale = TriMesh(exp_template_scale.points, \n                                 trilist=template_scale.trilist)\n    \n    exp_aspect_ratio = np.divide(*exp_template_scale.range()[:2])\n    r = GLRasterizer(width=template_im_width, height=template_im_width / exp_aspect_ratio)\n    r.set_model_matrix(rotate_model.h_matrix)\n    r.set_view_matrix(m2c_template_tr.h_matrix)\n\n    instance_image = r.rasterize_mesh_with_shape_image(model_exp_instance)[1]\n    template_scale_in_image = r.model_to_image_transform.apply(exp_template_scale)\n    \n    instance = cylindrical_snapping(instance_image, template_scale_in_image, exp_template_scale, u_t_shape_image,\n                                u_t_shape_index_image, verbose=False)\n    r._opengl.stop()\n    return instance",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "example_sample = generate_sensible_model_instance()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "%matplotlib inline\n\nexample_sample.view()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Real Image"
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "DATASETS_PATH = Path('/vol/atlas/databases')\nTDDFA_PATH = DATASETS_PATH / '3ddfa'",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": true,
    "trusted": true
   },
   "cell_type": "code",
   "source": "ibug_image_paths = list(filter(lambda x: x.stem != \"image_092_01\", mio.image_paths('/vol/atlas/databases/ibug')))\nafw_image_paths = list(mio.image_paths('/vol/atlas/databases/afw'))\nlfpw_image_paths = list(mio.image_paths('/vol/atlas/databases/lfpw/**/*'))",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "def get_3ddfa_data(path):\n    folder = path.parent.name\n    main_folder = folder.upper()\n    stem = path.stem\n    if folder in {'testset', 'trainset'}:\n        folder = Path(*path.parts[-3:-1])\n        main_folder = folder.parent.name.upper()\n    if main_folder == 'LFPW':\n        stem = 'image_{}_{}'.format(folder.name[:-3].lower(), \n                                    path.stem.split('_')[-1])\n    \n    image_original = mio.import_image(DATASETS_PATH / '{}/{}'.format(folder, path.name))\n    image_3ddfa = mio.import_image(TDDFA_PATH / '300W-3D/{}/{}.jpg'.format(main_folder, stem))\n\n    fitted_mesh = loadmat(str(TDDFA_PATH / '300W-3D-Face/{}/{}.mat'.format(main_folder, stem)))['Fitted_Face']\n    fit_info = loadmat(str(TDDFA_PATH / '300W-3D/{}/{}.mat'.format(main_folder, stem)))\n    print(fit_info['Pose_Para'])\n    \n    fitted_3d_mesh, fitted_2d_mesh =  convert_3ddfa_to_original(fitted_mesh, fit_info, \n                                                                image_3ddfa, image_original,\n                                                                original_template.trilist)\n    return fitted_3d_mesh, fitted_2d_mesh, image_original",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "def fit_3ddfa_images(path):\n    fitted_3d_mesh, fitted_2d_mesh, image_original = get_3ddfa_data(path)\n    \n    view_t, c_t, proj_t = retrieve_camera_matrix(image_original, fitted_2d_mesh, fitted_3d_mesh)\n    if fitted_3d_mesh.n_points != np.count_nonzero(trimmed_mask):\n        fitted_3d_mesh = fitted_3d_mesh.from_mask(trimmed_mask)\n        fitted_2d_mesh = fitted_2d_mesh.from_mask(trimmed_mask)\n        \n    rasterizer = GLRasterizer(height=image_original.height, \n                          width=image_original.width, \n                          view_matrix=view_t.h_matrix,\n                          projection_matrix=proj_t.h_matrix)\n\n    real_instance_image = rasterizer.rasterize_mesh_with_shape_image(fitted_3d_mesh)[1]\n\n    projected_fitted_3d_mesh = fitted_3d_mesh.project_barycentric_coordinates(*ts_to_ut_bary)\n    projected_fitted_3d_mesh = TriMesh(projected_fitted_3d_mesh.points, \n                                       trilist=template_scale.trilist)\n    projected_template_in_image = rasterizer.model_to_image_transform.apply(projected_fitted_3d_mesh)\n    rasterizer._opengl.stop()\n    \n    instance = cylindrical_snapping(real_instance_image, \n                                projected_template_in_image, \n                                projected_fitted_3d_mesh, \n                                u_t_shape_image, u_t_shape_index_image, \n                                verbose=False)\n    return instance",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "fitted_3d_mesh, fitted_2d_mesh, image_original = get_3ddfa_data(ibug_image_paths[87])\n# Rotate to upright to remove roll\nupright_theta = np.arctan2(*AlignmentSimilarity(fitted_3d_mesh, \n                                                original_template).h_matrix[:2, 0][::-1])\nimage_original, roll_remove_tr = image_original.rotate_ccw_about_centre(upright_theta, \n                                                                        degrees=False, \n                                                                        return_transform=True)\nfitted_2d_mesh = roll_remove_tr.pseudoinverse().apply(fitted_2d_mesh)\n\nview_t, c_t, proj_t = retrieve_camera_matrix(image_original, fitted_2d_mesh, fitted_3d_mesh)\nif fitted_3d_mesh.n_points != np.count_nonzero(trimmed_mask):\n    fitted_2d_mesh = fitted_2d_mesh.from_mask(trimmed_mask)\n    fitted_3d_mesh = fitted_3d_mesh.from_mask(trimmed_mask)\n\nrasterizer = GLRasterizer(height=image_original.height, \n                          width=image_original.width, \n                          view_matrix=view_t.h_matrix,\n                          projection_matrix=proj_t.h_matrix)\nprint(view_t.compose_before(proj_t).apply(fitted_3d_mesh).bounds())\nreal_instance_image = rasterizer.rasterize_mesh_with_shape_image(fitted_3d_mesh)[1]\n\nprojected_fitted_3d_mesh = fitted_3d_mesh.project_barycentric_coordinates(*ts_to_ut_bary)\nprojected_fitted_3d_mesh = TriMesh(projected_fitted_3d_mesh.points, \n                                   trilist=template_scale.trilist)\nprojected_template_in_image = rasterizer.model_to_image_transform.apply(projected_fitted_3d_mesh)\nrasterizer._opengl.stop()\n\ninstance = cylindrical_snapping(real_instance_image, \n                                projected_template_in_image, \n                                projected_fitted_3d_mesh, \n                                u_t_shape_image, u_t_shape_index_image, \n                                verbose=False)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "image_original.view()\nclip_to_image_transform(image_original.width, image_original.height).apply(a).view(marker_size=1, figure_size=(15, 15))",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# %matplotlib inline\nreal_instance_image.view(channels=0, figure_size=[5, 5])                  # X-axis, dark->light\nreal_instance_image.view(channels=1, figure_size=[5, 5], new_figure=True) # Y-axis, bottom dark->top light\nreal_instance_image.view(channels=2, figure_size=[5, 5], new_figure=True) # Z-axis, nose light",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# %matplotlib inline\nimage_original.view()\nfitted_2d_mesh.view(marker_size=1)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# %matplotlib inline\nimage_original.view()\nprojected_template_in_image.view(marker_size=1)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# Crap Sampling\nu_t_shape_image.from_vector(image_original.sample(projected_template_in_image.points)).view()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# Good Sampling\nu_t_shape_image.from_vector(image_original.sample(instance)).view()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "tr = ThinPlateSplines(u_t_shape_image.landmarks['gt'].lms,\n                      image_original.landmarks['PTS'].lms)\nimage_original.warp_to_mask(u_t_shape_image.mask, tr, \n                            warp_landmarks=False).view()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": true
   },
   "cell_type": "markdown",
   "source": "# Build Model"
  },
  {
   "metadata": {
    "collapsed": true,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# real_image_shapes = mio.import_pickle('/mnt/data/lfpw_dense_shapes_150_ref.pkl')",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "real_image_shapes = [fit_3ddfa_images(p) for p in print_progress(lfpw_image_paths)]",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "samples = [generate_sensible_model_instance(template_im_width=u_t_im_width) \n           for _ in print_progress(range(500))]",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "mouth_samples = [generate_sensible_model_instance(template_im_width=u_t_im_width,\n                                                  max_exp_ind=3, n_simultaneous_exp=1,\n                                                  min_exp_intensity=-0.9, max_exp_intensity=-0.1)\n                 for _ in print_progress(range(100))]",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": true,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# mio.export_pickle(real_image_shapes, '/mnt/data/lfpw_dense_shapes_150_ref.pkl')",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "# visualize_pointclouds(lfpw_shapes)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": true,
    "trusted": true
   },
   "cell_type": "code",
   "source": "shape_model = PCAModel(align_shapes(real_image_shapes + samples + mouth_samples))",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "visualize_shape_model(shape_model, n_parameters=10)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "(shape_model.eigenvalues_cumulative_ratio() < 0.98).sum()",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": true,
    "trusted": true
   },
   "cell_type": "code",
   "source": "shape_model.trim_components(n_components=30)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "reference_frame = MaskedImage.init_blank(u_t_shape_image.shape, \n                                         mask=u_t_shape_image.mask)\nreference_frame.landmarks['source'] = u_t_shape_image.landmarks['gt']\n\nmio.export_pickle(reference_frame,\n                  '/vol/atlas/homes/pts08/iccv2015/3d/model/neutral_template_10.pkl.gz',\n                 overwrite=True)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": false,
    "trusted": true
   },
   "cell_type": "code",
   "source": "mio.export_pickle(shape_model,\n                  '/vol/atlas/homes/pts08/iccv2015/3d/model/shape_model_10_30.pkl.gz', \n                  overwrite=True)",
   "execution_count": null,
   "outputs": []
  },
  {
   "metadata": {
    "collapsed": true,
    "trusted": true
   },
   "cell_type": "code",
   "source": "",
   "execution_count": null,
   "outputs": []
  }
 ],
 "metadata": {
  "language_info": {
   "file_extension": ".py",
   "name": "python",
   "version": "3.5.1",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "mimetype": "text/x-python"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}