Create a time-lapse video from a collection of selfies

time-lapse.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7dc0e063",
   "metadata": {},
   "source": [
    "# Time-Lapse\n",
    "\n",
    "Aswin van Woudenberg (https://www.aswinvanwoudenberg.com | https://github.com/afvanwoudenberg)\n",
    "\n",
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ff2135a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import os\n",
    "import glob\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import mediapipe as mp\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from os import listdir\n",
    "from os.path import isfile, join"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84f78ae8",
   "metadata": {},
   "source": [
    "## Defining constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8b4c5495-db89-43aa-a590-bd3adf059d13",
   "metadata": {},
   "outputs": [],
   "source": [
    "mp_face_mesh = mp.solutions.face_mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0501b089",
   "metadata": {},
   "outputs": [],
   "source": [
    "IMG_PATH = \"selfies/\" # the input directory\n",
    "OUTPUT_PATH = \"output/\" # also used for temporary files\n",
    "VIDEO_NAME = 'video.avi' # the output filename\n",
    "FPS = 3 # frames per second\n",
    "\n",
    "NOSE_TIP_LANDMARK = 1\n",
    "LEFTMOST_LANDMARK = 234\n",
    "RIGHTMOST_LANDMARK = 454"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32538af8",
   "metadata": {},
   "source": [
    "## Delete old files from previous runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b896c6d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "files = glob.glob(os.path.join(OUTPUT_PATH, \"*\"))\n",
    "for f in files:\n",
    "    os.remove(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5df2d13a",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "29711bc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_pixel_coord(image, landmark):\n",
    "    # convert landmark to pixel coordinates\n",
    "    [height, width, _] = image.shape\n",
    "    return int(landmark.x * width), int(landmark.y * height)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3cc7cd87",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_landmarks(path):\n",
    "    # find all files in directory\n",
    "    filenames = [f for f in listdir(path) if isfile(join(path, f))]\n",
    "    filenames.sort()\n",
    "    \n",
    "    # create an empty dataframe\n",
    "    columns = {\n",
    "        \"file\": str(), \n",
    "        \"nose_tip_x\": int(), \"nose_tip_y\": int(), \n",
    "        \"leftmost_x\": int(), \"leftmost_y\": int(), \n",
    "        \"rightmost_x\": int(), \"rightmost_y\": int(),\n",
    "        \"width\": int(), \"height\": int()\n",
    "    }\n",
    "    df = pd.DataFrame(columns, index=[])\n",
    "    \n",
    "    # find the landmarks' pixel coordinates\n",
    "    with mp_face_mesh.FaceMesh(static_image_mode=True, \n",
    "                max_num_faces=1, refine_landmarks=True, \n",
    "                min_detection_confidence=0.5) as face_mesh:\n",
    "        for file in filenames:\n",
    "            image = cv2.imread(os.path.join(path, file))\n",
    "            results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n",
    "\n",
    "            if not len(results.multi_face_landmarks) == 1:\n",
    "                # detected less or more than one face -> skip image\n",
    "                continue\n",
    "            face_landmarks = results.multi_face_landmarks[0]\n",
    "            nose_tip_x, nose_tip_y = to_pixel_coord(image, face_landmarks.landmark[NOSE_TIP_LANDMARK])\n",
    "            leftmost_x, leftmost_y = to_pixel_coord(image, face_landmarks.landmark[LEFTMOST_LANDMARK])\n",
    "            rightmost_x, rightmost_y = to_pixel_coord(image, face_landmarks.landmark[RIGHTMOST_LANDMARK])\n",
    "            [height, width, _] = image.shape\n",
    "            landmarks_xy = [file, nose_tip_x, nose_tip_y, leftmost_x, leftmost_y, rightmost_x, rightmost_y, width, height]\n",
    "            df = pd.concat([df, pd.DataFrame([landmarks_xy], columns=list(columns.keys()))], ignore_index=True)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "88a1c2da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def scale_image(filename_input, filename_output, factor):\n",
    "    # read image from disk\n",
    "    image = cv2.imread(filename_input)\n",
    "    \n",
    "    (height, width) = image.shape[:2]\n",
    "\n",
    "    res = cv2.resize(image, (int(width * factor), int(height * factor)), interpolation = cv2.INTER_CUBIC)\n",
    "  \n",
    "    # write image back to disk.\n",
    "    cv2.imwrite(filename_output, res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "62c50b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate_image(filename_input, filename_output, x, y):\n",
    "    # if the shift is (x, y) then the translation matrix would be\n",
    "    # M = [1 0 x]\n",
    "    #     [0 1 y]\n",
    "    M = np.float32([[1, 0, x], [0, 1, y]])\n",
    "    \n",
    "    # read image from disk.\n",
    "    image = cv2.imread(filename_input)\n",
    "    (rows, cols) = image.shape[:2]\n",
    "  \n",
    "    # warpAffine does appropriate shifting given the translation matrix.\n",
    "    res = cv2.warpAffine(image, M, (cols, rows))\n",
    "  \n",
    "    # write image back to disk.\n",
    "    cv2.imwrite(filename_output, res)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "130ca24a",
   "metadata": {},
   "source": [
    "## Process images"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "740d6690",
   "metadata": {},
   "source": [
    "### Find landmarks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c0f19b74",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO: Created TensorFlow Lite XNNPACK delegate for CPU.\n"
     ]
    }
   ],
   "source": [
    "df = read_landmarks(IMG_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b4e96f6",
   "metadata": {},
   "source": [
    "### Scale images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9318dea1",
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_face_size = int(df.rightmost_x.mean()) - int(df.leftmost_x.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "60586210",
   "metadata": {},
   "outputs": [],
   "source": [
    "for _, row in df.iterrows():\n",
    "    filename = row['file']\n",
    "    face_size = row['rightmost_x'] - row['leftmost_x']\n",
    "    scale_image(os.path.join(IMG_PATH, filename), os.path.join(OUTPUT_PATH, filename), mean_face_size / face_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5c097da7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_landmarks(OUTPUT_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63c28f47",
   "metadata": {},
   "source": [
    "### Translate images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a880f57b",
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_x = int(df.nose_tip_x.mean())\n",
    "mean_y = int(df.nose_tip_y.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5286820c",
   "metadata": {},
   "outputs": [],
   "source": [
    "crop_left = 0\n",
    "crop_right = 0\n",
    "crop_top = 0\n",
    "crop_bottom = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "452de652",
   "metadata": {},
   "outputs": [],
   "source": [
    "for _, row in df.iterrows():\n",
    "    filename = row['file']\n",
    "    shift_x = mean_x - row['nose_tip_x']\n",
    "    shift_y = mean_y - row['nose_tip_y']\n",
    "    translate_image(os.path.join(OUTPUT_PATH, filename), os.path.join(OUTPUT_PATH, filename), shift_x, shift_y)\n",
    "    \n",
    "    if shift_x > 0 and shift_x > crop_left:\n",
    "        crop_left = shift_x\n",
    "    elif shift_x < 0 and abs(shift_x) > crop_right:\n",
    "        crop_right = abs(shift_x)\n",
    "    elif shift_y > 0 and shift_y > crop_top:\n",
    "        crop_top = shift_y\n",
    "    elif shift_y < 0 and abs(shift_y) > crop_bottom:\n",
    "        crop_bottom = abs(shift_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71e4c8de",
   "metadata": {},
   "source": [
    "### Crop images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6fb3052a",
   "metadata": {},
   "outputs": [],
   "source": [
    "min_width = df.width.min()\n",
    "min_height = df.height.min()\n",
    "\n",
    "for _, row in df.iterrows():\n",
    "    filename = row['file']\n",
    "    image = cv2.imread(os.path.join(OUTPUT_PATH, filename))\n",
    "    (rows, cols) = image.shape[:2]\n",
    "    res = image[crop_top:min_height, crop_left:min_width]\n",
    "    cv2.imwrite(os.path.join(OUTPUT_PATH, filename), res)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4d83259",
   "metadata": {},
   "source": [
    "## Creating video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cb695294",
   "metadata": {},
   "outputs": [],
   "source": [
    "images = [img for img in os.listdir(OUTPUT_PATH)]\n",
    "images.sort()\n",
    "frame = cv2.imread(os.path.join(OUTPUT_PATH, images[0]))\n",
    "height, width, layers = frame.shape\n",
    "\n",
    "video = cv2.VideoWriter(os.path.join(OUTPUT_PATH, VIDEO_NAME), 0, FPS, (width, height))\n",
    "\n",
    "for image in images:\n",
    "    video.write(cv2.imread(os.path.join(OUTPUT_PATH, image)))\n",
    "\n",
    "cv2.destroyAllWindows()\n",
    "video.release()\n",
    "\n",
    "for image in images:\n",
    "    os.remove(os.path.join(OUTPUT_PATH, image))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0233977-a2ff-4cf8-a407-22c1559f7767",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}