A utility written in Ruby to download video from M3U8 files. Requires ffmpeg and the Clipboard gem.

download_m3u8.rb

# A utility written in Ruby to download video from M3U8 files.
#
# From Wikipedia:
#
# An M3U file is a plain text file that specifies the locations of one or more
# media files. The file is saved with the "m3u" filename extension if the text
# is encoded in the local system's default non-Unicode encoding (e.g., a
# Windows codepage), or with the "m3u8" extension if the text is UTF-8 encoded.

require 'optparse'
require 'clipboard'
require 'pathname'
require 'fileutils'
require 'uri'

options = Hash.new
OptionParser.new do |opts|
  opts.banner = "Usage: ruby download_from_manifest.rb [options]"
  opts.separator ""

  opts.on("-fFILE", "--file=FILE",
          "Name of file") do |v|

    options[:file] = v
  end

  opts.on("-d", "--dir=DIRECTORY",
          "Directory to store video",
          "(Default: .)") do |v|

    options[:dir] = v
  end
end.parse!

# SETUP: Ensure we have a video file name
unless options[:file]
  puts "Please pass in a name for the video ending with a .mp4 extension."
  exit(0)
end

tmp_file  = options[:file] + "-tmp"
dir_path  = Pathname.new(Dir.pwd)

if (options[:dir])
  dir_path = dir_path.join(options[:dir])
end

tmp_path  = dir_path.join(tmp_file)

# SETUP: Ensure we have a fresh tmp directory.
unless File.directory?(tmp_path)
  FileUtils.mkdir(tmp_path)
end

master_manifest_file = "master-manifest.txt"
master_manifest_path = tmp_path.join(master_manifest_file)

video_manifest_file  = "video-manifest.txt"
video_manifest_path = tmp_path.join(video_manifest_file)

ffmpeg_manifest = "ffmpeg-manifest.txt"
ffmpeg_manifest_path = tmp_path.join(ffmpeg_manifest)

video_path = dir_path.join(options[:file])

# STEP1: Open up the network tab in your browser. Find the video
# =>     master manifest. It will have a file extension of
# =>     ".secure.m3u8". Once you've copied the curl command
# =>     you can run this file.

curl_cmd = Clipboard.paste

if curl_cmd && curl_cmd.match(/^curl\s/)
  # Remove newlines
  curl_cmd.gsub!(/\\\n/, "")

  # Remove session id
  curl_cmd.gsub!(/-H\s\'X-Playback-Session-Id: [A-Z0-9\-]+\'/, "")

  # Remove range
  curl_cmd.gsub!(/-H\s\'Range: bytes=[0-9]+-[0-9]+\'/, "")

  # Make sure to download file
  curl_cmd = "curl -s -o #{master_manifest_path} " << curl_cmd.partition(/curl\s/).last

  puts "Downloading master manifest file"

  `#{curl_cmd}`

  puts "Download complete. 😁"

  Clipboard.clear
else
  puts "Clipboard empty. Exiting."
  exit(0)
end

# STEP2: Now that we have the master-manifest file we can
# =>     download each of the individual .ts files that
# =>     make up the video.
# =>
# =>     To do that we need to parse the master-manifest file
# =>     for the url where we can find the video manifest file.
uri = URI(URI.extract(File.read(master_manifest_path), /http(s)/).first)
uri_path  = uri.path.gsub(/[0-9a-zA-Z_-]+\.m3u8$/, "")
base_uri  = uri.scheme + "://" + uri.host + uri_path
video_manifest_uri = uri.scheme + "://" + uri.host + uri.path

puts "Downloading video manifest file"

`curl -s -o #{video_manifest_path} #{video_manifest_uri}`

  puts "Download complete. 😁"

# STEP3: We need to download each of the individual .ts files
# =>     that make up the video.

ts_files  = File.read(video_manifest_path)
                .split(/\n/)
                .select { |line| line.match(/\.ts$/) }

puts "Downloading #{ts_files.length} video files..."

thread = Thread.new do
  thread = Thread.current
  thread[:done] = 0

  ts_files.each do |ts|
    `curl -s -o #{tmp_path}/#{ts} #{base_uri}#{ts}`
    thread[:done] = thread[:done] + 1
  end
end

until thread.join(3) do
  STDOUT.write "\rDownload Progress: #{thread[:done]} / #{ts_files.length}"
end


if Dir.glob("#{tmp_path}/*.ts").length != ts_files.length
  puts "We didn't download the expected number of video files."
  puts "Exiting..."
  exit(0)
end

puts "Everything looks good. Time to build the FFMPEG manifest file."

# STEP4: Build a manifest file for FFMPEG.

target = File.open(ffmpeg_manifest_path, "w")

target.truncate(0)

files = Dir.glob("#{tmp_path}/*.ts")

# Helper function we use to make sure the files are in order
def int_from(file)
  /[0-9]+\.ts$/.match(file)
               .to_s
               .gsub!(".ts", "")
               .to_i
end

# Sort the files
files = files.sort do |a, b|
  int_from(a) <=> int_from(b)
end

# Write each file to the ffmpeg manifest
files.each do |f|
  target.write("file '#{f}'\n")
end

target.close

# STEP4: Concatenate the files together into the final video

puts "Manifest file built. Concatenating files..."

`ffmpeg -hide_banner -nostats -v error -f concat -safe 0 -i #{ffmpeg_manifest_path} -c copy #{video_path}`

puts "Concatenation complete. 😁"

# STEP5: Cleanup

puts "Removing #{tmp_path}"

FileUtils.rm_rf(tmp_path)

puts "We're DONE! Enjoy! 😀"