Video player for Roblox on EditableImage

!RobloxVideoPlayer.md

This is a video player for Roblox on EditableImage

How to use

On http server

• Change SERVER_ADDRESS in VideoPlayer.py to whatever you want
• Install dependencies for VideoPlayer.py script: numpy and opencv
• Run VideoPlayer.py on your server
• Setup VideoId.txt files, which link to video file and contain asset id of audio uploaded to roblox

Inside Roblox

• Put VideoDownloader.lua into ServerScriptService as server Script
• Inside VideoDownloader, create module Config which has UrlBase set to you server address in format https://example.com/
• Put VideoPlayer.lua as ModuleScript to somewhere accessible on client
• Create RemoteEvent-s (RequestGetFrames, ResponseGetFrames) and RemoteFunction-s (InitVideo)
• On client, require VideoPlayer and execute VideoPlayer.Play("VideoId")

BadApple.txt

{"File": "BadApple2.mp4", "AudioId": "rbxassetid://16153789173"}

KillerBean.txt

{"File": "KillerBean_500x282.mp4", "AudioId": "rbxassetid://16239932830"}

VideoDownloader.lua

--!strict
--!native
-- server script

local ReplicatedStorage = game:GetService("ReplicatedStorage")
local HttpService = game:GetService("HttpService")

local Events = require(ReplicatedStorage.VideoPlayer.Events)
local Config = require(script.Config)



Events.InitVideo.OnServerInvoke = function(player : Player, videoName : string)
	local videoInfoString = HttpService:GetAsync(Config.UrlBase .. videoName .. "/info")
	return videoInfoString
end



local FRAMES_PER_BLOCK = 20*10 -- TODO: move to config, make it dependent on video framerate
local ONE_EVENT_BUFFER_LIMIT = 10^7

Events.RequestGetFrames.OnServerEvent:Connect(
	function(player : Player, videoName : string, blockIndex : number)
		local frameStart = blockIndex * FRAMES_PER_BLOCK
		local frameEnd = (blockIndex+1) * FRAMES_PER_BLOCK - 1
		local framesString = HttpService:GetAsync(Config.UrlBase .. videoName .. `/frame/{frameStart}-{frameEnd}`)
		for i = 1, #framesString, ONE_EVENT_BUFFER_LIMIT do
			local framesStringPart = framesString:sub(i, i+ONE_EVENT_BUFFER_LIMIT-1)
			local framesBufferPart = buffer.fromstring(framesStringPart)
			Events.ResponseGetFrames:FireClient(player, videoName, blockIndex, i-1, framesBufferPart)
		end
	end
)

VideoPlayer.lua

--!strict
--!native
-- module script

local VideoPlayer = {}

local ReplicatedStorage = game:GetService("ReplicatedStorage")
local SoundService = game:GetService("SoundService")
local RunService = game:GetService("RunService")
local HttpService = game:GetService("HttpService")

local Events = require(ReplicatedStorage.VideoPlayer.Events)



type VideoInfo = {
	Framerate : number,
	Resolution : Vector2,
	AudioId : string,
}

local FRAMES_PER_BLOCK = 20*10 -- TODO: move to config, make it dependent on video framerate

local pixels : {[string] : {[number] : buffer, Array : {number}}} = {}

local function getVideoInfo(videoName : string) : VideoInfo
	local videoInfoString = Events.InitVideo:InvokeServer(videoName)
	local videoInfo = HttpService:JSONDecode(videoInfoString)
	videoInfo.Resolution = Vector2.new(videoInfo.Width, videoInfo.Height)
	videoInfo.Width = nil
	videoInfo.Height = nil
	return videoInfo
end

local function unloadPixels(videoName : string, blockIndex : number)
	pixels[videoName][blockIndex] = nil
end

local function downloadPixels(videoName : string, blockIndex : number, resolution : Vector2)
	-- TODO: get resolution by video name from cached video info
	pixels[videoName][blockIndex] = buffer.create(resolution.X * resolution.Y * 4 * FRAMES_PER_BLOCK) -- TODO: separate "isDownloading" flag
	Events.RequestGetFrames:FireServer(videoName, blockIndex)
end

Events.ResponseGetFrames.OnClientEvent:Connect(function(videoName : string, blockIndex : number, startByteIndex : number, framesBufferPart : buffer)
	local framesBuffer = pixels[videoName][blockIndex]
	buffer.copy(framesBuffer, startByteIndex, framesBufferPart)
	--buffer.writestring(framesBuffer, startByteIndex, framesStringPart)
end)

local function getPixels(videoName : string, frameIndex : number, resolution : Vector2)
	-- TODO: get resolution by video name from cached video info
	local blockIndex = frameIndex // FRAMES_PER_BLOCK
	local frameInBlockIndex = frameIndex % FRAMES_PER_BLOCK
	local framesBuffer = pixels[videoName][blockIndex]
	if framesBuffer == nil then
		warn(`No frames for block {blockIndex}`)
		return
	end
	debug.profilebegin("Creating pixels array")
	local frameSize = resolution.X * resolution.Y * 4
	local startByteIndex = frameInBlockIndex * frameSize - 1
	local endByteIndex = startByteIndex + frameSize - 1
	for i = 1, frameSize do
		local byteIndex = startByteIndex + i
		local byte = buffer.readu8(framesBuffer, byteIndex)
		pixels[videoName].Array[i] = byte/255
	end
	debug.profileend()
end

local function onHeartbeat(audio : Sound, videoInfo : VideoInfo, videoName : string, editableImage : EditableImage)
	local frameIndex = math.round(audio.TimePosition * videoInfo.Framerate) + 1
	local blockIndex = frameIndex // FRAMES_PER_BLOCK

	debug.profilebegin("Unload/download pixels")
	unloadPixels(videoName, blockIndex-1)
	if not pixels[videoName][blockIndex+1] then -- TODO: separate "isDownloading" flag
		downloadPixels(videoName, blockIndex+1, videoInfo.Resolution)
	end
	debug.profileend()

	getPixels(videoName, frameIndex, videoInfo.Resolution)
	debug.profilebegin("Write pixels")
	editableImage:WritePixels(
		Vector2.zero,
		videoInfo.Resolution,
		pixels[videoName].Array
	)
	debug.profileend()
end

local function stopVideo(connection : RBXScriptConnection, videoName : string, editableImage : EditableImage, audio : Sound)
	connection:Disconnect()
	pixels[videoName] = nil
	editableImage:Destroy()
	audio:Destroy()
end

function VideoPlayer.Play(canvas : ImageLabel, videoName : string)
	
	local videoInfo = getVideoInfo(videoName)
	pixels[videoName] = {
		Array = table.create(videoInfo.Resolution.X * videoInfo.Resolution.Y * 4, 1)
	}
	downloadPixels(videoName, 0, videoInfo.Resolution)
	downloadPixels(videoName, 1, videoInfo.Resolution)
	task.wait(2) -- TODO: wait for initial pixels to download
	
	local editableImage = Instance.new("EditableImage")
	editableImage.Size = videoInfo.Resolution
	editableImage.Parent = canvas
	
	local audio = Instance.new("Sound")
	audio.Name = videoName
	audio.Volume = 0.15
	audio.SoundId = videoInfo.AudioId
	audio.Parent = SoundService
	
	local connection = RunService.Heartbeat:Connect(function()
		onHeartbeat(audio, videoInfo, videoName, editableImage)
	end)
	
	audio.Ended:Connect(function()
		stopVideo(connection, videoName, editableImage, audio)
	end)
	audio:GetPropertyChangedSignal("Parent"):Connect(function()
		if not audio.Parent then
			stopVideo(connection, videoName, editableImage, audio)
		end
	end)
	audio:Play()
	
end



return VideoPlayer

VideoPlayer.py

from http.server import BaseHTTPRequestHandler, HTTPServer
import cv2
import json
import numpy as np



SERVER_ADDRESS = ('', 51630)


class MyHttpHandler(BaseHTTPRequestHandler):

	def do_GET(self):
		path = self.path.split("/")
		video_id = path[1]
		command = path[2]

		with open(video_id+".txt") as f:
			video_info = json.loads(f.read())
		capture = cv2.VideoCapture(video_info["File"])
		if command == "info":
			self.send_response(200)
			self.end_headers()
			self.wfile.write(json.dumps({
				"Framerate": capture.get(cv2.CAP_PROP_FPS),
				"Width": capture.get(cv2.CAP_PROP_FRAME_WIDTH),
				"Height": capture.get(cv2.CAP_PROP_FRAME_HEIGHT),
				"AudioId": video_info["AudioId"],
			}).encode())
		elif command == "frame":
			[frames_start, frames_end] = path[3].split("-")
			frames_start = int(frames_start)
			frames_end = int(frames_end)
			capture.set(cv2.CAP_PROP_POS_FRAMES, frames_start)
			self.send_response(200)
			self.end_headers()
			for frame_index in range(frames_start, frames_end+1):
				isSuccess, frame = capture.read()
				frame = np.pad(frame, ((0,0),(0,0),(0,1)), mode='constant', constant_values=255)
				if isSuccess:
					self.wfile.write(frame.tobytes())
				else:
					print("ERROR GETTING FRAME", video_id, frame_index)
					break
		else:
			self.send_response(400)
			self.end_headers()
		capture.release()



server = HTTPServer(SERVER_ADDRESS, MyHttpHandler)
try:
	print("Serving...")
	server.serve_forever()
except KeyboardInterrupt:
	server.server_close()
	print("KeyboardInterrupt")