Skip to content

Instantly share code, notes, and snippets.

@svagionitis
Last active June 21, 2024 22:54
Show Gist options
  • Save svagionitis/9644441 to your computer and use it in GitHub Desktop.
Save svagionitis/9644441 to your computer and use it in GitHub Desktop.
Segment an mp4 file to HLS streaming files
#!/bin/bash
# Create an Iframe index from HLS segmented streams
# $1: Filename to be created
# $2: Location of segmented ts files
# Check how many arguments
if [ $# != 2 ]; then
echo "Usage: $0 [Input filename] [Location of segmented streams]"
exit 1;
fi
# Check if parameters have values
if [ -z "$1" ]; then
echo "No filename was given."
exit 1;
fi
if [ -z "$2" ]; then
echo "No location was given."
exit 1;
fi
# Check if location exists
if [ ! -d "${2}" ]; then
echo "Location '${2}' doesn't exist."
exit 1;
fi
IFRAME_FILE="IFRAME_${1}.m3u8"
rm -f ${IFRAME_FILE}
touch ${IFRAME_FILE}
echo "#EXTM3U" > ${IFRAME_FILE}
echo "#EXT-X-TARGETDURATION:" >> ${IFRAME_FILE}
echo "#EXT-X-VERSION:4" >> ${IFRAME_FILE}
echo "#EXT-X-I-FRAMES-ONLY" >> ${IFRAME_FILE}
# Find all segmented files and sort by number
SEGMENTED_FILES=$(find "$2" -name \*.ts | sort -V)
# Source http://www.cyberciti.biz/tips/handling-filenames-with-spaces-in-bash.html
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
for FILE in ${SEGMENTED_FILES}
do
# Put into parenthesis to make arrays
GET_NUM_IFRAMES=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I'))
GET_PTS_TIME=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_pts_time=\([0-9]*.[0-9]*\).*/\1/p"))
GET_PKT_SIZE=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_size=\([0-9]*\).*/\1/p"))
GET_PKT_POS=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_pos=\([0-9]*\).*/\1/p"))
GET_LAST_PTS_TIME=$(ffprobe -show_frames ${FILE} -of compact | tail -1 | sed -n "s/.*pkt_pts_time=\([0-9]*.[0-9]*\).*/\1/p")
for (( i = 0 ; i < ${#GET_NUM_IFRAMES[@]} ; i++ ))
do
RESULT=
if [ $(( $i + 1 )) -lt "${#GET_NUM_IFRAMES[@]}" ]; then
RESULT=$(echo "${GET_PTS_TIME[ (( $i + 1 )) ]} - ${GET_PTS_TIME[$i]}" | bc -l)
else
RESULT=$(echo "${GET_LAST_PTS_TIME} - ${GET_PTS_TIME[$i]}" | bc -l)
fi
echo "#EXTINF:${RESULT}" >> ${IFRAME_FILE}
echo "#EXT-X-BYTERANGE:${GET_PKT_SIZE[$i]}@${GET_PKT_POS[$i]}" >> ${IFRAME_FILE}
echo "${FILE}" >> ${IFRAME_FILE}
done
done
IFS=$SAVEIFS
echo "#EXT-X-ENDLIST" >> ${IFRAME_FILE}
#!/bin/bash
# Get the total duration of the video,
# reading the durations from the M3U8 list
# $1: Location of the M3U8 list
# Check how many arguments
if [ $# != 1 ]; then
echo "Usage: $0 [Location of M3U8 list]"
exit 1;
fi
# Check if parameters have values
if [ -z "${1}" ]; then
echo "No list was given."
exit 1;
fi
# Check if list exists
if [ ! -f "${1}" ]; then
echo "List '${1}' doesn't exist."
exit 1;
fi
# Get all the durations of the segmented streams and create an array
GET_DURATIONS=($(cat "${1}" | sed -n "s/^#EXTINF:\([0-9]*.[0-9]*\).*/\1/p"))
TOTAL_DURATION="0"
for (( i = 0 ; i < ${#GET_DURATIONS[@]} ; i++ ))
do
GET_DURATION_MSEC=$(echo "${GET_DURATIONS[$i]} * 1000" | bc -l)
TOTAL_DURATION=$(echo "${GET_DURATION_MSEC} + ${TOTAL_DURATION}" | bc -l)
done
echo "M3U8 List: "${1}" Duration ${TOTAL_DURATION} ms, $(echo "${TOTAL_DURATION} / 1000" | bc -l) sec, $(echo "${TOTAL_DURATION} / 1000 / 60" | bc -l) min"
#!/bin/bash
# Get the total duration of the stream
# by adding the duration of the segmented
# streams.
# $1: Location of the M3U8 list, where the segmented
# streams will read
# Check how many arguments
if [ $# != 1 ]; then
echo "Usage: $0 [Location of M3U8 list of segmented streams]"
exit 1;
fi
# Check if parameters have values
if [ -z "${1}" ]; then
echo "No location was given."
exit 1;
fi
# Check if file exists
if [ ! -f "${1}" ]; then
echo "Location of list '${1}' doesn't exist."
exit 1;
fi
DIRECTORY=$(dirname "${1}")
# Find all segmented files from the list and sort by number,
# if they are not sorted.
SEGMENTED_FILES=$(cat "${1}" | grep \.*.ts | sort -V)
# Source http://www.cyberciti.biz/tips/handling-filenames-with-spaces-in-bash.html
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
TOTAL_DURATION="0"
for FILE in ${SEGMENTED_FILES}
do
GET_DURATION_SEC=$(ffprobe -show_format "${DIRECTORY}"/"${FILE}" -of compact | sed -n "s/.*duration=\([0-9]*.[0-9]*\).*/\1/p")
GET_DURATION_MSEC=$(echo "${GET_DURATION_SEC} * 1000" | bc -l)
TOTAL_DURATION=$(echo "${GET_DURATION_MSEC} + ${TOTAL_DURATION}" | bc -l)
done
IFS=$SAVEIFS
echo "M3U8 List: "${1}" Duration ${TOTAL_DURATION} ms, $(echo "${TOTAL_DURATION} / 1000" | bc -l) sec, $(echo "${TOTAL_DURATION} / 1000 / 60" | bc -l) min"
#!/bin/bash
# Get the m3u8 lists from a file
# $1: A file which contains m3u8 lists, this
# could be a json or csv or anything else in the
# format "test.m3u8":
# $2: A prefix directory to the location of streams e.g /tmp/streams or /home/$USER/Downloads
# Check how many arguments
if [ $# != 2 ]; then
echo "Usage: $0 [Location of the file containing the M3U8 lists] [prefix to the location of streams. e.g /tmp/streams]"
exit 1;
fi
# Check if parameters have values
if [ -z "${1}" ]; then
echo "No file was given."
exit 1;
fi
if [ -z "${2}" ]; then
echo "No prefix was given."
exit 1;
fi
# Check if file exists
if [ ! -f "${1}" ]; then
echo "File '${1}' doesn't exist."
exit 1;
fi
# Find all segmented files from the list and sort by number,
# if they are not sorted.
M3U8_LISTS=$(cat "${1}" | sed -n "s/.*\"\(.*\.m3u8\)\"\:.*/\1/p")
# Source http://www.cyberciti.biz/tips/handling-filenames-with-spaces-in-bash.html
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
for list in $M3U8_LISTS
do
echo ""${2}"/"$list""
done
IFS=$SAVEIFS
#!/bin/bash
# This script segment a mp4 video file to HLS compatible streams using ffmpeg.
# See below for the parameters used in ffmpeg.
CPUs=$(grep -c ^processor /proc/cpuinfo)
# $1: Filename to be created
# $2: Bitrate
# $3: Resolution
# $4: M3U8 playlist
function CREATE_BASIC_VARIANT_M3U8_LIST {
BASIC_VARIANT_FILE="BASIC_VARIANT_${1}.m3u8"
rm -f ${BASIC_VARIANT_FILE}
touch ${BASIC_VARIANT_FILE}
echo "#EXTM3U" > ${BASIC_VARIANT_FILE}
echo "#EXT-X-VERSION:3" >> ${BASIC_VARIANT_FILE}
echo "#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=${2},CODECS=\"avc1.4d001f,mp4a.40.2\",RESOLUTION=${3}" >> ${BASIC_VARIANT_FILE}
echo "${4}" >> ${BASIC_VARIANT_FILE}
echo "#EXT-X-ENDLIST" >> ${BASIC_VARIANT_FILE}
}
# $1: Filename to be created
# $2: Location of segmented ts files
function CREATE_IFRAME_M3U8_LIST {
IFRAME_FILE="IFRAME_${1}.m3u8"
rm -f ${IFRAME_FILE}
touch ${IFRAME_FILE}
echo "#EXTM3U" > ${IFRAME_FILE}
echo "#EXT-X-TARGETDURATION:" >> ${IFRAME_FILE}
echo "#EXT-X-VERSION:4" >> ${IFRAME_FILE}
echo "#EXT-X-I-FRAMES-ONLY" >> ${IFRAME_FILE}
# Find all segmented files and sort by number
SEGMENTED_FILES=$(find "$2" -name \*.ts | sort -n)
# Source http://www.cyberciti.biz/tips/handling-filenames-with-spaces-in-bash.html
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
for FILE in ${SEGMENTED_FILES}
do
# Put into parenthesis to make arrays
GET_NUM_IFRAMES=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I'))
GET_PTS_TIME=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_pts_time=\([0-9]*.[0-9]*\).*/\1/p"))
GET_PKT_SIZE=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_size=\([0-9]*\).*/\1/p"))
GET_PKT_POS=($(ffprobe -show_frames ${FILE} -of compact | grep 'pict_type=I' | sed -n "s/.*pkt_pos=\([0-9]*\).*/\1/p"))
GET_LAST_PTS_TIME=$(ffprobe -show_frames ${FILE} -of compact | tail -1 | sed -n "s/.*pkt_pts_time=\([0-9]*.[0-9]*\).*/\1/p")
for (( i = 0 ; i < ${#GET_NUM_IFRAMES[@]} ; i++ ))
do
RESULT=
if [ $(( $i + 1 )) -lt "${#GET_NUM_IFRAMES[@]}" ]; then
RESULT=$(echo "${GET_PTS_TIME[ (( $i + 1 )) ]} - ${GET_PTS_TIME[$i]}" | bc -l)
else
RESULT=$(echo "${GET_LAST_PTS_TIME} - ${GET_PTS_TIME[$i]}" | bc -l)
fi
echo "#EXTINF:${RESULT}" >> ${IFRAME_FILE}
echo "#EXT-X-BYTERANGE:${GET_PKT_SIZE[$i]}@${GET_PKT_POS[$i]}" >> ${IFRAME_FILE}
echo "${FILE}" >> ${IFRAME_FILE}
done
done
IFS=$SAVEIFS
echo "#EXT-X-ENDLIST" >> ${IFRAME_FILE}
}
# Check how many arguments
if [ $# != 5 ]; then
echo "Usage: $0 [mp4 video] [demux: segment or hls] [output directory prefix] [hls duration of each segment stream] [single ts file(single_file) or not(multiple_files), byterange to .m3u8]"
exit 1;
fi
# Check if parameters have values
if [ -z "$1" ]; then
echo "No input video file was given."
exit 1;
fi
if [ -z "$2" ]; then
echo "No demux was given."
exit 1;
fi
if [ -z "$3" ]; then
echo "No output directory prefix was given."
exit 1;
fi
if [ -z "$4" ]; then
echo "No HLS duration was given."
exit 1;
fi
if [ -z "$5" ]; then
echo "Single file or not was not specified."
exit 1;
fi
FILE_INPUT="$1"
# Check if the file exists
if [ ! -f "${FILE_INPUT}" ]; then
echo "File '${FILE_INPUT}' doesn't exist."
exit 1;
fi
# Check if the file is MP4 by extention
if [ "${FILE_INPUT##*.}" != 'mp4' ]; then
echo "File '${MP4_FILE}' is not an mp4 video."
exit 1;
fi
DEMUX="$2"
# Check if the correct demux used.
if [ "$DEMUX" != 'segment' -a "$DEMUX" != 'hls' ]; then
echo "Demux '$DEMUX' is not correct. Use segment or hls."
exit 1;
fi
DIR_OUTPUT_PREFIX="$3"
HLS_DURATION="$4"
SINGLE_FILE="$5"
if [ "$SINGLE_FILE" != 'single_file' -a "$SINGLE_FILE" != 'multiple_files' ]; then
echo "Single file option '$SINGLE_FILE' is not correct. Use single_file or multiple_files."
exit 1;
fi
DIR_OUTPUT=""${DIR_OUTPUT_PREFIX}"-"${DEMUX}"-"${HLS_DURATION}"-"${SINGLE_FILE}""
# Check if directory exists
if [ ! -f "${DIR_OUTPUT}" ]; then
echo "Directory '${DIR_OUTPUT}' doesn't exist. Creating..."
mkdir -p "${DIR_OUTPUT}"
fi
FFPROBE_FORMAT_COMPACT=$(ffprobe -show_format "${FILE_INPUT}" -of compact)
GET_BITRATE=$(echo "${FFPROBE_FORMAT_COMPACT}" | sed -n "s/.*bit_rate=\([0-9]*\).*/\1/p")
GET_DURATION_SEC=$(echo "${FFPROBE_FORMAT_COMPACT}" | sed -n "s/.*duration=\([0-9]*.[0-9]*\).*/\1/p")
GET_DURATION_MSEC=$(echo "${GET_DURATION_SEC} * 1000" | bc -l)
FFPROBE_STREAMS_COMPACT=$(ffprobe -show_streams "${FILE_INPUT}" -of compact)
GET_WIDTH=$(echo "${FFPROBE_STREAMS_COMPACT}" | grep video | sed -n "s/.*width=\([0-9]*\).*/\1/p")
GET_HEIGHT=$(echo "${FFPROBE_STREAMS_COMPACT}" | grep video | sed -n "s/.*height=\([0-9]*\).*/\1/p")
GET_RESOLUTION="${GET_WIDTH}x${GET_HEIGHT}"
GET_VIDEO_CODEC_TAG_STRING=$(echo "${FFPROBE_STREAMS_COMPACT}" | grep video | sed -n "s/.*codec_tag_string=\([0-9A-Za-z]*\).*/\1/p")
GET_AUDIO_CODEC_TAG_STRING=$(echo "${FFPROBE_STREAMS_COMPACT}" | grep audio | sed -n "s/.*codec_tag_string=\([0-9A-Za-z]*\).*/\1/p")
MP4_FILE_NOPATH="${FILE_INPUT##*/}"
FILENAME="${MP4_FILE_NOPATH%.*}"
COMMAND_OUTPUT=''
COMMAND_EXECUTED=''
if [ "${DEMUX}" == "segment" ]; then
COMMAND_OUTPUT=$(ffmpeg -i "${FILE_INPUT}" -threads "${CPUs}" -codec copy -map 0 -f "${DEMUX}" -vbsf h264_mp4toannexb -flags -global_header -segment_format mpegts -segment_list "${DIR_OUTPUT}"/"${FILENAME}".m3u8 -segment_list_flags +live -segment_time "${HLS_DURATION}" "${DIR_OUTPUT}"/"${FILENAME}"-%03d.ts)
elif [ "${DEMUX}" == "hls" ]; then
if [ "${SINGLE_FILE}" == "multiple_files" ]; then
COMMAND_OUTPUT=$(ffmpeg -i "${FILE_INPUT}" -threads "${CPUs}" -codec copy -map 0 -f "${DEMUX}" -vbsf h264_mp4toannexb -flags -global_header -hls_time "${HLS_DURATION}" -hls_list_size 0 "${DIR_OUTPUT}"/"${FILENAME}".m3u8)
elif [ "${SINGLE_FILE}" == "single_file" ]; then
COMMAND_OUTPUT=$(ffmpeg -i "${FILE_INPUT}" -threads "${CPUs}" -codec copy -map 0 -f "${DEMUX}" -vbsf h264_mp4toannexb -flags -global_header -hls_time "${HLS_DURATION}" -hls_list_size 0 -hls_flags single_file "${DIR_OUTPUT}"/"${FILENAME}".m3u8)
fi
fi
echo "Command output: '$COMMAND_OUTPUT'"
echo "Bitrate: ${GET_BITRATE} Resolution: ${GET_RESOLUTION}"
CREATE_BASIC_VARIANT_M3U8_LIST ${FILENAME} ${GET_BITRATE} ${GET_RESOLUTION} "${DIR_OUTPUT}"/"${FILENAME}".m3u8
# ffmpeg parameter explanation
# -i "${FILE_INPUT}": Give the input video stream to segment.
# -codec copy: Copy the stream without reencoding. Documentation https://www.ffmpeg.org/ffmpeg.html#Stream-copy.
# -map 0: Map ALL streams from the video file to output. See documentation of -map https://www.ffmpeg.org/ffmpeg.html#Advanced-options.
# -f "$MUXER": Select muxer segment or hls. For segment Documentation https://www.ffmpeg.org/ffmpeg.html#Main-options and https://ffmpeg.org/ffmpeg-formats.html#segment_002c-stream_005fsegment_002c-ssegment. For hls Documentation https://ffmpeg.org/ffmpeg-formats.html#hls-1.
# -vbsf h264_mp4toannexb: Bitstream filter h264_mp4toannexb. Documentation https://www.ffmpeg.org/ffmpeg-bitstream-filters.html#h264_005fmp4toannexb.
# -flags: Unset flags. Documentation https://www.ffmpeg.org/ffmpeg-all.html#Codec-Options.
# -global_header: Don't place global headers in extradata. Instead place in every keyframe. Documentation under flags https://www.ffmpeg.org/ffmpeg-all.html#Codec-Options.
# See http://www.ffmpeg.org/ffmpeg-formats.html#segment for segmenter settings.
# -segment_format: Override the inner container format, by default it is guessed by the filename extension. Here is mpegts
# -segment_list: Generate also a listfile named name. If not specified no listfile is generated.
# -segment_time: Set segment duration to time, the value must be a duration specification. Default value is "2". Here is 10.
# See https://ffmpeg.org/ffmpeg-formats.html#hls-1 for hls settings
# -hls_time: Set segment duration to time, the value must be a duration specification. Default value is "2". Here is 10.
# -hls_list_size: Set the maximum number of playlist entries. If set to 0 the list file will contain all the segments. Default value is 5.
# -hls_flags single_file: If this flag is set, the muxer will store all segments in a single MPEG-TS file, and will use byte ranges in the playlist. HLS playlists generated with this way will have the version number 4. Documentation https://www.ffmpeg.org/ffmpeg-formats.html#Options-2
@umar-veeve
Copy link

Should not this command produce clips with avc1 (annexb) bitstream instead of h.264?

COMMAND_OUTPUT=$(ffmpeg -i "${FILE_INPUT}" -threads "${CPUs}" -codec copy -map 0 -f "${DEMUX}" -vbsf h264_mp4toannexb -flags -global_header -hls_time "${HLS_DURATION}" -hls_list_size 0 "${DIR_OUTPUT}"/"${FILENAME}".m3u8)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment