Skip to content

Instantly share code, notes, and snippets.

@sgqy
Last active May 20, 2023 16:58
Show Gist options
  • Save sgqy/f551e8630d6f9d55684e60706e6d41fb to your computer and use it in GitHub Desktop.
Save sgqy/f551e8630d6f9d55684e60706e6d41fb to your computer and use it in GitHub Desktop.
multicore minterpolate in ffmpeg
#!/bin/bash
### multicore minterpolate in ffmpeg
# just slice & process & concat
# the concat points between sliced may be weird, but it just works
# default args
ff=/usr/bin/ffmpeg
fps=60000/1001
nut=00:00:10
enc=libx264
task=$(nproc --all)
ff_args=( -hide_banner -loglevel level+warning -stats -y )
#############################
err_args () {
printf "\nUsage: %s <-i in_file> [args...] <out_file>\n" "$(basename "$0")"
printf "\n"
printf "Arguments:\n"
printf "\t-r N\tTarget FPS\t\t\t[%s]\n" "${fps}"
printf "\t-s T\tStart point of source video\n"
printf "\t-t T\tEnd point of source video\n"
printf "\t-e str\tEncoder name\t\t\t[%s]\n" "${enc}"
printf "\t-P N\tParallel count. 0 is max\t[%s]\n" "${task}"
printf "\t-C T\tVideo duration of parallel unit\t[%s]\n" "${nut}"
printf "\t-F str\tAlternative ffmpeg path\t\t[%s]\n" "${ff}"
exit 1
}
clean () {
rm -f _cut.mp4
rm -f _s_?????.mp4
rm -f _m__s_?????.mp4
rm -f _list.txt
}
while getopts 'i:r:s:t:e:P:C:F:' OPTION; do
case $OPTION in
i)
in=$OPTARG # input file
;;
r)
fps=$OPTARG # target fps, fraction is also OK
;;
s)
ss=$OPTARG # begin time of input
;;
t)
to=$OPTARG # end time of input
;;
e)
enc=$OPTARG # encoder
;;
P)
task=$((OPTARG)) # process count, xargs use max when it is 0
;;
C)
nut=$OPTARG # parallel block length
;;
F)
ff=$OPTARG # ffmpeg binary
;;
*)
err_args
;;
esac
done
shift $((OPTIND - 1))
# set target file
out="$1"
# select a temp file to slice: original file or cut from original
temp="${in}"
# check i/o files
if [ -z "${in}" ]; then
echo '*** No input file'
err_args
fi
if [ -z "${out}" ]; then
echo '*** No output file'
err_args
fi
# set duration to convert and temp file
duration=()
if [ -n "${ss}" ]; then
duration+=(-ss)
duration+=("${ss}")
temp=_cut.mp4
fi
if [ -n "${to}" ]; then
duration+=(-to)
duration+=("${to}")
temp=_cut.mp4
fi
#############################
set -x
clean
# use origin file or cut a part from origin file
if [ "${temp}" == _cut.mp4 ]; then
"${ff}" "${ff_args[@]}" "${duration[@]}" -i "${in}" -c copy "${temp}"
fi
# make slices for parallel use
"${ff}" "${ff_args[@]}" -i "${temp}" -c copy -f segment -segment_time "${nut}" '_s_%05d.mp4'
# convert the slices ### this progress may be extremely SLOW
find . -name '_s_?????.mp4' | sed 's@^.*/@@g' | xargs -i -t -P ${task} "${ff}" "${ff_args[@]}" -i '{}' -vf "minterpolate=fps=${fps}" -c:a copy -c:v "${enc}" '_m_{}'
# make the list
printf "file '%s'\n" ./_m__s_?????.mp4 > _list.txt
# concat the result
"${ff}" "${ff_args[@]}" -f concat -safe 0 -i _list.txt -c copy "${out}"
# clean dir
clean
@majal
Copy link

majal commented Dec 27, 2022

Awesome work on this one.

@cclecle
Copy link

cclecle commented Apr 2, 2023

super nice idea !

question:
Is making slice temp file really necessary ?
Can't you get list of key_frames, pick interresting ones to cut the full dataset and then use them to process the transcoding ?

Here is a sample script I did for you to achieve that:

#!/usr/bin/env bash


##############################################################
# Demo script to cut a movie file in segment using ffprobe
# while trying to keep closer to keyframes
#
# Usage: > ./cufile.sh <FilePath>
#
#########################################

set -e # stop on first error

get_numberOfCPU(){
	getconf _NPROCESSORS_ONLN
}
get_keyFrames(){
	local FileName="$1"
	ffprobe -select_streams v -skip_frame nokey -v quiet -of default=noprint_wrappers=1 -show_entries frame=pkt_pts_time "$FileName" | sed 's/pkt_pts_time=//'
}
get_NearestTime(){
	local RequestedTime="$1"
	shift #to reference dataset array at first arg position
	local DataSet=("$@"); IFS=: # IFS is used to delimit array
	# using awk to process the array in one command (using awk in a loop is too slow)
	echo "$RequestedTime" | awk -v dataset="${DataSet[*]}"  '{
		n_dataset=split(dataset, _dataset, "[:]")
		for (i=1; i<=length(_dataset); i++)
		{
			if ($RequestedTime < _dataset[i])
			{
				print _dataset[i]
				exit
			}
		}
		print "/!\ Can not find a NearestTime" > "/dev/stderr"
		exit 1
	}'
}

File="$1"
NumberOfCut=$(get_numberOfCPU)

echo "[INFO] Expected number of cuts: $NumberOfCut"

echo "[INFO] Extracting Keyframes..."
mapfile -t keyframes < <(get_keyFrames "$File")
echo "[INFO] Done"

FirstValue=0
LastValue="${keyframes[-1]}"

echo "[INFO] Cutting file from $FirstValue sec to $LastValue sec, in $NumberOfCut segments."
SegmentSize=$(awk -v var1="$LastValue" -v var2="$NumberOfCut"  'BEGIN { print  ( var1 / var2 ) }')
echo "[INFO] Computed approx segments size is $SegmentSize sec"

SegmentKeyFrames=(0)

for (( CutIndex=1; CutIndex<"$NumberOfCut"; CutIndex++ )); do
	TargetKeyFrame=$(awk -v var1="$CutIndex" -v var2="$SegmentSize"  'BEGIN { print  ( var1*var2 ) }')
	echo "[DEBUG] TargetKeyFrame is $TargetKeyFrame"
	RealKeyFrame=$(get_NearestTime "$TargetKeyFrame" "${keyframes[@]}")
	echo "[DEBUG] RealKeyFrame is $RealKeyFrame"
	SegmentKeyFrames+=("$RealKeyFrame")
done
SegmentKeyFrames+=("$LastValue")

echo "[INFO] Done."

echo "[INFO] Final SegmentKeyFrames are:"
for i in "${SegmentKeyFrames[@]}"; do
	echo "[INFO] $i"
done

Sample output:

[INFO] Expected number of cuts: 8
[INFO] Extracting Keyframes...
[INFO] Done
[INFO] Cutting file from 0 sec to 6525.727000 sec, in 8 segments.
[INFO] Computed approx segments size is 815.716 sec
[DEBUG] TargetKeyFrame is 815.716
[DEBUG] RealKeyFrame is 818.985000
[DEBUG] TargetKeyFrame is 1631.43
[DEBUG] RealKeyFrame is 1631.713000
[DEBUG] TargetKeyFrame is 2447.15
[DEBUG] RealKeyFrame is 2452.283000
[DEBUG] TargetKeyFrame is 3262.86
[DEBUG] RealKeyFrame is 3262.968000
[DEBUG] TargetKeyFrame is 4078.58
[DEBUG] RealKeyFrame is 4083.830000
[DEBUG] TargetKeyFrame is 4894.3
[DEBUG] RealKeyFrame is 4898.852000
[DEBUG] TargetKeyFrame is 5710.01
[DEBUG] RealKeyFrame is 5710.580000
[INFO] Done.
[INFO] Final SegmentKeyFrames are:
[INFO] 0
[INFO] 818.985000
[INFO] 1631.713000
[INFO] 2452.283000
[INFO] 3262.968000
[INFO] 4083.830000
[INFO] 4898.852000
[INFO] 5710.580000
[INFO] 6525.727000

Note: I guess target frame rate must be double (or any integer multiplier) from the original to get a clear transcoding..

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment