Skip to content

Instantly share code, notes, and snippets.

@bitingsock
Last active December 20, 2024 15:49
Show Gist options
  • Save bitingsock/17d90e3deeb35b5f75e55adb19098f58 to your computer and use it in GitHub Desktop.
Save bitingsock/17d90e3deeb35b5f75e55adb19098f58 to your computer and use it in GitHub Desktop.
Precache the next entry in your playlist if it is a network source by downloading it to a temp file ahead of time. Change Line 20 to temp directory. It will delete the directory on exit.
----------------------
-- #example ytdl_preload.conf
-- # make sure lines do not have trailing whitespace
-- # ytdl_opt has no sanity check and should be formatted exactly how it would appear in yt-dlp CLI, they are split into a key/value pair on whitespace
-- # at least on Windows, do not escape '\' in temp, just us a single one for each divider
-- #temp=R:\ytdltest
-- #ytdl_opt1=-r 50k
-- #ytdl_opt2=-N 5
-- #ytdl_opt#=etc
----------------------
local nextIndex
local caught = true
-- local pop = false
local ytdl = "yt-dlp"
local utils = require 'mp.utils'
local options = require 'mp.options'
local opts = {
temp = "R:\\ytdl",
ytdl_opt1 = "",
ytdl_opt2 = "",
ytdl_opt3 = "",
ytdl_opt4 = "",
ytdl_opt5 = "",
ytdl_opt6 = "",
ytdl_opt7 = "",
ytdl_opt8 = "",
ytdl_opt9 = "",
}
options.read_options(opts, "ytdl_preload")
local additionalOpts = {}
for k, v in pairs(opts) do
if k:find("ytdl_opt%d") and v ~= "" then
additionalOpts[k] = v
-- print("entry")
-- print(k .. v)
end
end
local cachePath = opts.temp
local chapter_list = {}
local json = ""
local filesToDelete = {}
local function exists(file)
local ok, err, code = os.rename(file, file)
if not ok then
if code == 13 then -- Permission denied, but it exists
return true
end
end
return ok, err
end
local function useNewLoadfile()
for _, c in pairs(mp.get_property_native("command-list")) do
if c["name"] == "loadfile" then
for _, a in pairs(c["args"]) do
if a["name"] == "index" then
return true
end
end
end
end
end
--from ytdl_hook
local function time_to_secs(time_string)
local ret
local a, b, c = time_string:match("(%d+):(%d%d?):(%d%d)")
if a ~= nil then
ret = (a * 3600 + b * 60 + c)
else
a, b = time_string:match("(%d%d?):(%d%d)")
if a ~= nil then
ret = (a * 60 + b)
end
end
return ret
end
local function extract_chapters(data, video_length)
local ret = {}
for line in data:gmatch("[^\r\n]+") do
local time = time_to_secs(line)
if time and (time < video_length) then
table.insert(ret, { time = time, title = line })
end
end
table.sort(ret, function(a, b) return a.time < b.time end)
return ret
end
local function chapters()
if json.chapters then
for i = 1, #json.chapters do
local chapter = json.chapters[i]
local title = chapter.title or ""
if title == "" then
title = string.format('Chapter %02d', i)
end
table.insert(chapter_list, { time = chapter.start_time, title = title })
end
elseif not (json.description == nil) and not (json.duration == nil) then
chapter_list = extract_chapters(json.description, json.duration)
end
end
--end ytdl_hook
local title = ""
local fVideo = ""
local fAudio = ""
local function load_files(dtitle, destination, audio, wait)
if wait then
if exists(destination .. ".mka") then
print("---wait success: found mka---")
audio = "audio-file=" .. destination .. '.mka,'
else
print("---could not find mka after wait, audio may be missing---")
end
end
-- if audio ~= "" then
-- table.insert(filesToDelete, destination .. ".mka")
-- end
-- table.insert(filesToDelete, destination .. ".mkv")
dtitle = dtitle:gsub("-" .. ("[%w_-]"):rep(11) .. "$", "")
dtitle = dtitle:gsub("^" .. ("%d"):rep(10) .. "%-", "")
if useNewLoadfile() then
mp.commandv("loadfile", destination .. ".mkv", "append", -1,
audio .. 'force-media-title="' .. dtitle .. '",demuxer-max-back-bytes=1MiB,demuxer-max-bytes=3MiB,ytdl=no')
else
mp.commandv("loadfile", destination .. ".mkv", "append",
audio .. 'force-media-title="' .. dtitle .. '",demuxer-max-back-bytes=1MiB,demuxer-max-bytes=3MiB,ytdl=no') --,sub-file="..destination..".en.vtt") --in case they are not set up to autoload
end
mp.commandv("playlist_move", mp.get_property("playlist-count") - 1, nextIndex)
mp.commandv("playlist_remove", nextIndex + 1)
caught = true
title = ""
-- pop = true
end
local listenID = ""
local function listener(event)
if not caught and event.prefix == mp.get_script_name() and string.find(event.text, listenID) then
local destination = string.match(event.text, "%[download%] Destination: (.+).mkv") or
string.match(event.text, "%[download%] (.+).mkv has already been downloaded")
-- if destination then print("---"..cachePath) end;
if destination and string.find(destination, string.gsub(cachePath, '~/', '')) then
-- print(listenID)
mp.unregister_event(listener)
_, title = utils.split_path(destination)
local audio = ""
if fAudio == "" then
load_files(title, destination, audio, false)
else
if exists(destination .. ".mka") then
audio = "audio-file=" .. destination .. '.mka,'
load_files(title, destination, audio, false)
else
print("---expected mka but could not find it, waiting for 2 seconds---")
mp.add_timeout(2, function()
load_files(title, destination, audio, true)
end)
end
end
end
end
end
--from ytdl_hook
mp.add_hook("on_preloaded", 10, function()
if string.find(mp.get_property("path"), cachePath) then
chapters()
if next(chapter_list) ~= nil then
mp.set_property_native("chapter-list", chapter_list)
chapter_list = {}
json = ""
end
end
end)
--end ytdl_hook
function dump(o)
if type(o) == 'table' then
local s = '{ '
for k, v in pairs(o) do
if type(k) ~= 'number' then k = '"' .. k .. '"' end
s = s .. '[' .. k .. '] = ' .. dump(v) .. ','
end
return s .. '} '
else
return tostring(o)
end
end
local function addOPTS(old)
for k, v in pairs(additionalOpts) do
-- print(k)
if string.find(v, "%s") then
for l, w in string.gmatch(v, "([-%w]+) (.+)") do
table.insert(old, l)
table.insert(old, w)
end
else
table.insert(old, v)
end
end
-- print(dump(old))
return old
end
local AudioDownloadHandle = {}
local VideoDownloadHandle = {}
local JsonDownloadHandle = {}
local function download_files(id, success, result, error)
if result.killed_by_us then
return
end
local jfile = cachePath .. "/" .. id .. ".json"
local jfileIO = io.open(jfile, "w")
jfileIO:write(result.stdout)
jfileIO:close()
json = utils.parse_json(result.stdout)
-- print(dump(json))
if json.requested_downloads[1].requested_formats ~= nil then
local args = { ytdl, "--no-continue", "-q", "-f", fAudio, "--restrict-filenames", "--no-playlist", "--no-part",
"-o", cachePath .. "/" .. id .. "-%(title)s-%(id)s.mka", "--load-info-json", jfile }
args = addOPTS(args)
AudioDownloadHandle = mp.command_native_async({
name = "subprocess",
args = args,
playback_only = false
}, function()
end)
else
fAudio = ""
fVideo = fVideo:gsub("bestvideo", "best")
fVideo = fVideo:gsub("bv", "best")
end
local args = { ytdl, "--no-continue", "-f", fVideo .. '/best', "--restrict-filenames", "--no-playlist",
"--no-part", "-o", cachePath .. "/" .. id .. "-%(title)s-%(id)s.mkv", "--load-info-json", jfile }
args = addOPTS(args)
VideoDownloadHandle = mp.command_native_async({
name = "subprocess",
args = args,
playback_only = false
}, function()
end)
end
local function DL()
local index = tonumber(mp.get_property("playlist-pos"))
if mp.get_property("playlist/" .. index .. "/filename"):find("/videos$") and mp.get_property("playlist/" .. index + 1 .. "/filename"):find("/shorts$") then
return
end
if tonumber(mp.get_property("playlist-pos-1")) > 0 and mp.get_property("playlist-pos-1") ~= mp.get_property("playlist-count") then
nextIndex = index + 1
local nextFile = mp.get_property("playlist/" .. nextIndex .. "/filename")
if nextFile and caught and nextFile:find("://", 0, false) then
caught = false
mp.enable_messages("info")
mp.register_event("log-message", listener)
local ytFormat = mp.get_property("ytdl-format")
fVideo = string.match(ytFormat, '(.+)%+.+//?') or 'bestvideo'
fAudio = string.match(ytFormat, '.+%+(.+)//?') or 'bestaudio'
-- print("start"..nextFile)
listenID = tostring(os.time())
local args = { ytdl, "--dump-single-json", "--no-simulate", "--skip-download",
"--restrict-filenames",
"--no-playlist", "--sub-lang", "en", "--write-sub", "--no-part", "-o",
cachePath .. "/" .. listenID .. "-%(title)s-%(id)s.%(ext)s", nextFile }
args = addOPTS(args)
-- print(dump(args))
table.insert(filesToDelete, listenID)
JsonDownloadHandle = mp.command_native_async({
name = "subprocess",
args = args,
capture_stdout = true,
capture_stderr = true,
playback_only = false
}, function(...)
download_files(listenID, ...)
end)
end
end
end
local function clearCache()
-- print(pop)
--if pop == true then
mp.abort_async_command(AudioDownloadHandle)
mp.abort_async_command(VideoDownloadHandle)
mp.abort_async_command(JsonDownloadHandle)
-- for k, v in pairs(filesToDelete) do
-- print("remove: " .. v)
-- os.remove(v)
-- end
local ftd = io.open(cachePath .. "/temp.files", "a")
for k, v in pairs(filesToDelete) do
ftd:write(v .. "\n")
if package.config:sub(1, 1) ~= '/' then
os.execute('del /Q /F "' .. cachePath .. "\\" .. v .. '*"')
else
os.execute('rm -f ' .. cachePath .. "/" .. v .. "*")
end
end
ftd:close()
print('clear')
mp.command("quit")
--end
end
mp.add_hook("on_unload", 50, function()
-- mp.abort_async_command(AudioDownloadHandle)
-- mp.abort_async_command(VideoDownloadHandle)
mp.abort_async_command(JsonDownloadHandle)
mp.unregister_event(listener)
caught = true
listenID = "resetYtdlPreloadListener"
-- print(listenID)
end)
local skipInitial
mp.observe_property("playlist-count", "number", function()
if skipInitial then
DL()
else
skipInitial = true
end
end)
--from ytdl_hook
local platform_is_windows = (package.config:sub(1, 1) == "\\")
local o = {
exclude = "",
try_ytdl_first = false,
use_manifests = false,
all_formats = false,
force_all_formats = true,
ytdl_path = "",
}
local paths_to_search = { "yt-dlp", "yt-dlp_x86", "youtube-dl" }
--local options = require 'mp.options'
options.read_options(o, "ytdl_hook")
local separator = platform_is_windows and ";" or ":"
if o.ytdl_path:match("[^" .. separator .. "]") then
paths_to_search = {}
for path in o.ytdl_path:gmatch("[^" .. separator .. "]+") do
table.insert(paths_to_search, path)
end
end
local function exec(args)
local ret = mp.command_native({
name = "subprocess",
args = args,
capture_stdout = true,
capture_stderr = true
})
return ret.status, ret.stdout, ret, ret.killed_by_us
end
local msg = require 'mp.msg'
local command = {}
for _, path in pairs(paths_to_search) do
-- search for youtube-dl in mpv's config dir
local exesuf = platform_is_windows and ".exe" or ""
local ytdl_cmd = mp.find_config_file(path .. exesuf)
if ytdl_cmd then
msg.verbose("Found youtube-dl at: " .. ytdl_cmd)
ytdl = ytdl_cmd
break
else
msg.verbose("No youtube-dl found with path " .. path .. exesuf .. " in config directories")
--search in PATH
command[1] = path
es, json, result, aborted = exec(command)
if result.error_string == "init" then
msg.verbose("youtube-dl with path " .. path .. exesuf .. " not found in PATH or not enough permissions")
else
msg.verbose("Found youtube-dl with path " .. path .. exesuf .. " in PATH")
ytdl = path
break
end
end
end
--end ytdl_hook
mp.register_event("start-file", DL)
mp.register_event("shutdown", clearCache)
local ftd = io.open(cachePath .. "/temp.files", "r")
while ftd ~= nil do
local line = ftd:read()
if line == nil or line == "" then
ftd:close()
io.open(cachePath .. "/temp.files", "w"):close()
break
end
-- print("DEL::"..line)
if package.config:sub(1, 1) ~= '/' then
os.execute('del /Q /F "' .. cachePath .. "\\" .. line .. '*" >nul 2>nul')
else
os.execute('rm -f ' .. cachePath .. "/" .. line .. "* &> /dev/null")
end
end
@lamborghinipth
Copy link

lamborghinipth commented Jun 23, 2024

@bitingsock

The simplest way is to just change line 260 to what you want, instead of local ytFormat = mp.get_property("ytdl-format") you can put local ytFormat = "bestvideo[height<=?1080][vcodec*=?avc]+bestaudio/best"

otherwise you could make a new profile whose condition looks for either the script file or the --scripts option if you are loading it that way

Thanks, the first method works exactly as what I want: 1080p and AVC codec.

otherwise you could make a new profile whose condition looks for either the script file or the --scripts option if you are loading it that way

Could you guide me step by step the second way. Where can I create a profile and what does profile look like? I almost copy and paste so I maybe do not know how to do, sorry about that.

@bitingsock
Copy link
Author

profiles are the part in you mpv.conf that look like:
[quality-youtube]
profile-cond=string.find(path, 'youtube') ~= nil or string.find(path, 'youtu.be') ~= nil or string.find(path, 'yt.be') ~= nil

depending on how are you enabling/disabling ytdl-preload we can write some logic into profile-cond to look for it. how are you enabling/disabling?

@lamborghinipth
Copy link

lamborghinipth commented Jun 24, 2024

profiles are the part in you mpv.conf that look like: [quality-youtube] profile-cond=string.find(path, 'youtube') ~= nil or string.find(path, 'youtu.be') ~= nil or string.find(path, 'yt.be') ~= nil

depending on how are you enabling/disabling ytdl-preload we can write some logic into profile-cond to look for it. how are you enabling/disabling?

Frankly, I don't know how to switch profiles or enable/ disable profiles. Because these are what I collected from many sources. Of course, I would like to use different profiles for different purposes.
Btw, if it is not easy to config, so forget it. The first way works like a charm. I will come back after learning to use profiles.

How about saving Unicode filename? Is it easy to implement?

@bitingsock
Copy link
Author

if I disable ytdl-preload.lua

I just mean the script, if you are disabling, I would need to know how

I think filenames should be easy to fix. I'll do it eventually, make sure to watch the repo:
https://github.com/bitingsock/ytdl-preload

@lamborghinipth
Copy link

@bitingsock

I think filenames should be easy to fix. I'll do it eventually, make sure to watch the repo: https://github.com/bitingsock/ytdl-preload

Yes, I am following and check everyday.
I am looking forward to hearing from you.

Another question: how to download more than 1 subtitle language? I use dual sub in MPV when watching Youtube link

====================
Video for testing: https://youtu.be/Pv0iVoSZzN8

My output: 100

However, there are only EN subtitles. I want download Vietnamese and Chinese subtitles. Is it possible?

@bitingsock
Copy link
Author

@lamborghinipth
ok both fixes are live over on the repo. subLangs is now a script-opt you can set in ytdl-preload.conf
for you, something like subLangs=vi,zh.* is what you want

@lamborghinipth
Copy link

lamborghinipth commented Jun 28, 2024

@bitingsock

@lamborghinipth ok both fixes are live over on the repo. subLangs is now a script-opt you can set in ytdl-preload.conf for you, something like subLangs=vi,zh.* is what you want

Thanks for quick update.
I have downloaded and tested https://github.com/bitingsock/ytdl-preload

There maybe 3 issues:

1.1 - The video is downloaded but MPV still connect to Youtube link and plays online video instead of playing downloaded file
1.2 - And of course, it also does not delete temp downloaded files after I close MPV window.
1.3 - Further more, the script will run and download video again whenever I click to play in MPV Playlist. As you see, 3 videos ..._I_Built_100_Wells_In_Africa-mwKJfNYwvm8 were downloaded: 2 vids at 720p and 1 at 1080p (I do not understand why we have 2 different qualities here though I didn't change quality of Youtube video, everything is default).
100


2 - Only 2 subLangs can be downloaded at a time. I cannot download the third language or more.
When I set subLangs=en,vi,zh -> the script only downloads EN and VI sub (I did't find ZH sub)
while if I set subLangs=en,vi,zh.*, the script downloads EN sub and all exts of ZH sub (I didn't find VI sub)

100


3 - Downloaded Date
I see all downloaded date are today in previous version of ytdl-preload.lua
The date makes me confuse now in latest version.
100

Please double check!

@bitingsock
Copy link
Author

bitingsock commented Jun 28, 2024

1: I have confirmed it working on windows and ubuntu 22.04. Something is wrong with your setup somehow. Please provide an mpv log. That said, it is intentional that the video is downloaded separately for each instance, as someone may have multiple mpv instances open downloading playlists.
2: that video has no native VI subs, only the auto-translated ones, you'll have to learn a little regex to get the exact sub selection you want. If you want auto translated subs you will need to include --write-auto-subs as a ytdl_opts# in ytdl-preload.conf
3: I have no control over the date, that's a yt-dlp issue. I highly doubt it was different in the previous version here, nothing about that part changed.

@lamborghinipth
Copy link

lamborghinipth commented Jul 3, 2024

@bitingsock
Sorry for late response
I have tried your latest version and it work perfectly now.

2: that video has no native VI subs, only the auto-translated ones, you'll have to learn a little regex to get the exact sub selection you want. If you want auto translated subs you will need to include --write-auto-subs as a ytdl_opts# in ytdl-preload.conf

This is my yt-dlp.conf

-o "dl/%(title)s.%(ext)s"
--format=bv+ba/b
--no-check-certificates
--trim-filenames 100
--yes-playlist
--write-sub
--write-auto-sub

This is ytdl-preload.conf - your latest version

#ytdl_preload.conf
# make sure lines do not have trailing whitespace
# ytdl_opt has no sanity check and should be formatted exactly how it would appear in yt-dlp CLI, they are split into a key/value pair on whitespace
# at least on Windows, do not escape '\' in temp, just us a single one for each divider

temp=C:\MPV\dl
subLangs=en,vi
ytdl_opt1=--extractor-args youtube:player_client=android,web
ytdl_opt2=--throttled-rate 1M
ytdl_opt3=-N 5
#ytdl_opt4=

I have --write-auto-sub in yt-dlp.conf so do I need to write this into ytdl-preload.conf?

@bitingsock
Copy link
Author

I would think the yt-dlp.conf should be fine but you probably want your subLangs to be something like:
en,vi,vi-en,zh-[^-]*,zh-.*-en
this should get:

english,
vietnamese (not auto translated),
vietnamese (auto translated from english),
chinese (simplified and traditional, not auto translated),
and
chinese (simplified and traditional, auto translated from english)

respectively

@lamborghinipth
Copy link

lamborghinipth commented Jul 3, 2024

@bitingsock

I would think the yt-dlp.conf should be fine but you probably want your subLangs to be something like: en,vi,vi-en,zh-[^-]*,zh-.*-en this should get:

english, vietnamese (not auto translated), vietnamese (auto translated from english), chinese (simplified and traditional, not auto translated), and chinese (simplified and traditional, auto translated from english)

respectively

Thanks a lot! It works as I hope
100

Some small questions:
1 - Can I disable line 6 in ytdl-preload.conf temp=C:\MPV\dl (you disabled this by default) and I also see it in line 21 of ytdl-preload.lua?

2 - How to download the first link of playlist?
100

@bitingsock
Copy link
Author

1: if you have edited line 21 of ytdl-preload.lua, you do not need it in ytdl-preload.conf. However, you probably should use the conf setting so that if you update the lua you don't have to worry about changing line 21 every time
2: I will have to add that feature

@lamborghinipth
Copy link

1: if you have edited line 21 of ytdl-preload.lua, you do not need it in ytdl-preload.conf. However, you probably should use the conf setting so that if you update the lua you don't have to worry about changing line 21 every time

Got it. I will disable the line 21 in .lua file

2: I will have to add that feature

Yes, I think we should force downloading video whenever the link is pasted to MPV player instead of playing directly from the internet
I am also using ytproxy that make ytdlp download in max speed by opening more connections like the way Internet Download Manager (IDM) does. It takes only few seconds to download 100MB video in 720p/1080p quality.

@bitingsock
Copy link
Author

  • don't disable it, it still needs to be there for the option to load into.
  • if the video is DASH, that's what -N 5 does, gets 5 segments at a time

@lamborghinipth
Copy link

@bitingsock Ok, got it.

Thanks for your help. Please notify when you update the feature download the first video instead of from the second video.
I am looking forward ^^

@lamborghinipth
Copy link

@bitingsock is there any good news?

@bitingsock
Copy link
Author

ok did it

@lamborghinipth
Copy link

@bitingsock

ok did it

I haven't checked latest released version (last week). What's new?

@bitingsock
Copy link
Author

it will now download the first video when you start playing the last one

@lamborghinipth
Copy link

it will now download the first video when you start playing the last one

What happens if I only paste one Youtube link to MPV? Does it download video immediately or after playing the first time?

@bitingsock
Copy link
Author

no, that's a bit out of the scope of "preloading" in my opinion. Personally, I do have a chain of scripts that does that for me but it does also add considerable startup time for the video, like triple or more.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment