Work in progress: create a markov chain for a video to generate new frames

video_markov.py

import os
import io
import cv2
import math
import matplotlib.pyplot as plt
import pandas as pd
from tensorflow.keras.preprocessing import image_dataset_from_directory
import numpy as np
from keras.utils import np_utils
from skimage.transform import resize
from typing import List, Tuple
from skimage.io import imread
import PIL.Image
from keras.models import Sequential
from keras.applications.vgg16 import VGG16
from keras.layers import Dense, InputLayer, Dropout
import pickle
import random
import imageio

def create_frames():
    """split mp4 into frames and save them locally"""
    count = 0
    videoFile = "tom_and_jerry.mp4"
    cap = cv2.VideoCapture(videoFile)   # capturing the video from the given path
    frameRate = cap.get(5) #frame rate
    x=1
    while(cap.isOpened()):
        frameId = cap.get(1) #current frame number
        ret, frame = cap.read()
        if (ret != True):
            break
        if (frameId % math.floor(frameRate) == 0):
            filename ="frame%d.jpg" % count;count+=1
            cv2.imwrite("frames/" + filename, frame)
    cap.release()
    print ("Done!")

def get_image_data() -> List:
    """load image data"""
    return [imread("frames/"+frame) for frame in sorted(os.listdir("frames"))]

def make_pairs(data: List) -> Tuple[str, str]:
    """
    Get pairs of the pixel in the last frame compared to the next frame
    Pixel data is converted into strings so it can be stored as a dictionary key
    """
    for x in range(len(data[0])):
        for y in range(len(data[0][0])):
            for i in range(len(data) - 1):
                pixel_1 = np.array(data[i][x][y], dtype=np.str).tolist()
                pixel_2 = np.array(data[i+1][x][y], dtype=np.str).tolist()
                yield (",".join(pixel_1), ",".join(pixel_2))

def make_markov_chain(pairs: Tuple[str, str]) -> dict:
    """Initialize the markov chain data structure"""
    ret = {}
    for pixel_1, pixel_2 in pairs:
        if pixel_1 in ret.keys():
            ret[pixel_1].append(pixel_2)
        else:
            ret[pixel_1] = [pixel_2]

    return ret

def generate_pixel(last_pixel, markov_chain, default):
    key = ",".join([str(i) for i in last_pixel])
    hits = markov_chain.get(key)
    if not hits:
        hits = default
    
    choice = random.randrange(len(hits))
    ret = hits[choice].split(",")
    return [int(i) for i in ret]

def generate_frame(last_frame, markov_chain, default):
    size_x = len(last_frame)
    size_y = len(last_frame[0])
    new_frame = []
    for x in range(size_x):
        new_row = []
        for y in range(size_y):
            new_row.append(generate_pixel(last_frame[x][y], markov_chain, default))
        
        new_frame.append(new_row)
    
    return new_frame


if __name__ == "__main__":
    # create_frames()
    # img = plt.imread('frames/frame0.jpg')   # reading image using its name
    # plt.imshow(img)
    # plt.show()
    # data = get_image_data()
    # pairs = make_pairs(data[:10])
    # markov_chain = make_markov_chain(pairs)
    # pickle.dump(markov_chain, open("markov_chain.pickle", "ab"))
    # markov_chain = pickle.load(open('markov_chain.pickle', 'rb'))
    # data = data[:10]
    # for _ in range(10):
    #     data.append(generate_frame(data[-1], markov_chain, list(markov_chain.keys())))

    # pickle.dump(data, open("new_frames.pickle", "ab"))
    data = pickle.load(open('new_frames.pickle', 'rb'))
    # im = imageio.get_reader(data, '.gif')
    imageio.mimsave('result.gif', data)