Download the HOPL IV sessions from SlidesLive

slideslive-dl.swift

#!/usr/bin/env swift
//
// Copyright 2021 Steven Brunwasser
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
// Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
import Foundation
import PDFKit
import Cocoa





// MARK: - Extensions

extension NSRegularExpression {
    convenience init(_ pattern: String) {
        try! self.init(pattern: pattern, options: [])
    }
}

extension String {
    func substrings(matching pattern: String, in range: Range<String.Index>) -> [Substring] {
        let regex = try! NSRegularExpression(pattern: pattern, options: .anchorsMatchLines)
        return regex.matches(in: self, options: [], range: NSRange(range, in: self)).map { match in
            self[Range(match.range(at: 0), in: self)!]
        }
    }

    func substrings(matching pattern: String) -> [Substring] {
        self.substrings(matching: pattern, in: self.startIndex..<self.endIndex)
    }

    func substringGroups(matching pattern: String, in range: Range<String.Index>) -> [[Substring]] {
        let regex = try! NSRegularExpression(pattern: pattern, options: .anchorsMatchLines)
        return regex.matches(in: self, options: [], range: NSRange(range, in: self)).map { match in
            (0..<match.numberOfRanges).map { index in
                let r = match.range(at: index)
                guard r.location != NSNotFound, let range = Range(match.range(at: index), in: self) else {
                    return Substring("")
                }
                return self[range]
            }
        }
    }

    func substringGroups(matching pattern: String) -> [[Substring]] {
        self.substringGroups(matching: pattern, in: self.startIndex..<self.endIndex)
    }
}

extension Substring {
    func substrings(matching pattern: String) -> [Substring] {
        self.base.substrings(matching: pattern, in: self.startIndex..<self.endIndex)
    }

    func substringGroups(matching pattern: String) -> [[Substring]] {
        self.base.substringGroups(matching: pattern, in: self.startIndex..<self.endIndex)
    }
}

extension Array {
    func appending(_ element: Element) -> Self {
        var array = self
        array.append(element)
        return array
    }

    func appending<S>(contentsOf sequence: S) -> Self where S: Sequence, S.Element == Element {
        var array = self
        array.append(contentsOf: sequence)
        return array
    }
}

extension Sequence {    
    func asyncMap<T>(_ transform: @escaping (Element) async throws -> T) async rethrows -> [T] {
        try await withThrowingTaskGroup(of: (T, Int).self, returning: [T].self) { taskGroup in
            var sequence = AnySequence(self)
            var result: [T] = []
            while case let elements = Array(sequence.prefix(50)), !elements.isEmpty {
                defer { sequence = AnySequence(sequence.dropFirst(elements.count)) }
                var collect: [(value: T, index: Int)] = []
                for (element, index) in zip(elements, elements.indices) {
                    taskGroup.addTask {
                        (try await transform(element), index)
                    }
                }
                for try await element in taskGroup {
                    collect.append(element)
                }
                collect.sort(by: { lhs, rhs in lhs.index < rhs.index })
                result.append(contentsOf: collect.map(\.value))
            }
            return result
        }
    }
}

extension NSImage {
    func compressedImage(factor: Double = 0) -> NSImage {
        return self.tiffRepresentation
            .flatMap(NSBitmapImageRep.init(data:))
            .flatMap({ $0.representation(using: .jpeg, properties: [.compressionFactor : factor]) })
            .flatMap(NSImage.init(data:)) ?? self
    }
}

extension PDFDocument {
    convenience init?(images: [NSImage]) {
        guard !images.isEmpty else {
            self.init()
            return
        }
        let pages = images.compactMap(PDFPage.init(image:))
        guard pages.count == images.count,
        let data = pages.first!.dataRepresentation else {
            return nil
        }
        self.init(data: data)
        zip(pages, pages.indices).dropFirst().forEach { page, index in
            self.insert(page, at: index)
        }
    }

    convenience init(contentsOfImagesAt urls: [URL]) throws {
        self.init(images: try urls.map { url in
            guard let image = NSImage(contentsOf: url) else {
                throw GenericError(message: "Cannot open image at \(url.path)")
            }
            return image.compressedImage()
        })!
    }
}





// MARK: - Utilities

extension FileHandle: TextOutputStream {
    static var stderr: FileHandle {
        get { FileHandle.standardError }
        set { }
    }
    public func write(_ string: String) {
        self.write(string.data(using: .utf8)!)
    }
}

actor Console {
    private let handle: FileHandle
    private var context: String?
    private var contextTotal: Int = 0
    private var contextValue: Int = 0

    init(handle: FileHandle) {
        self.handle = handle
        self.context = nil
    }

    func print(_ content: Any, retln: Bool = false, newln: Bool = true) {
        if retln {
            let clr = "\r                                                    \r"
            self.handle.write(clr.data(using: .utf8)!)
        }
        self.handle.write("\(content)".data(using: .utf8)!)
        if newln {
            self.handle.write("\n".data(using: .utf8)!)
        }
    }

    func beginContext(_ message: String) {
        guard self.context == nil else { fatalError() }
        self.context = message
        self.contextTotal = 0
        self.contextValue = 0
        self.print("\(message)...", retln: false, newln: false)
    }

    func reportProgress(total: Int) {
        guard let _ = self.context else { return }
        self.contextTotal = total
    }

    func reportProgress(increment: Int = 1) {
        guard let message = self.context else { return }
        self.contextValue += increment
        self.print("\(message)... \(self.contextValue) of \(self.contextTotal)", retln: true, newln: false)
    }

    func endContext() {
        guard let message = self.context else { return }
        self.print("\(message)... done.", retln: true, newln: true)
        self.context = nil
        self.contextTotal = 0
        self.contextValue = 0
    }

    func failContext() {
        guard let message = self.context else { return }
        self.print("\(message)... failed.", retln: true, newln: true)
        self.context = nil
    }
}

let stdout = Console(handle: .standardOutput)
let stderr = Console(handle: .standardError)

func withStatus<T>(_ message: String, do block: () async throws -> T) async rethrows -> T {
    await stdout.beginContext(message)
    do {
        let result = try await block()
        await stdout.endContext()
        return result
    } catch {
        await stdout.failContext()
        throw error
    }
}

struct GenericError: Error, CustomStringConvertible {
    var message: String

    var description: String {
        "GenericError: \(message)"
    }
}

struct ExitError: Error, CustomStringConvertible {
    var process: String
    var exitCode: Int32
    var stderror: String?

    init(process: Process, stderr: Pipe?) {
        self.process = process.launchPath!
        self.exitCode = process.terminationStatus
        try? stderr?.fileHandleForWriting.close()
        guard let stderr = stderr?.fileHandleForReading else {
            self.stderror = nil
            return
        }
        let errdata = try? stderr.readToEnd()
        self.stderror = errdata.flatMap { String(data: $0, encoding: .utf8) }
    }
    
    var description: String {
        let message = "ExitError: \(process) failed with exit code \(exitCode)"
        return stderror.map { stderror in
            "\(message)\n\t\(stderror.replacingOccurrences(of: "\n", with: "\n\t"))"
        } ?? message
    }
}





// MARK: - M3U Decoder

struct M3UDecoder {
    private var content: String

    init(data: Data) {
        self.content = String(data: data, encoding: .utf8)!
    }

    func contains(_ key: CodingKey) -> Bool {
        !self.content.substrings(matching: "^#\(key):?.*$").isEmpty
    }

    func decode(_ type: Bool.Type, forKey key: CodingKey) throws -> Bool {
        !self.content.substrings(matching: "^#\(key):$").isEmpty
    }

    func decode(_ type: Int.Type, forKey key: CodingKey) throws -> Int {
        guard let group = self.content.substringGroups(matching: "^#\(key.stringValue):(\\d+)$").first else {
            let context = DecodingError.Context(codingPath: [], debugDescription: "Keyed Int not found.")
            throw DecodingError.keyNotFound(key, context)
        }
        guard group.count == 2, case let content = group[1] else {
            let context = DecodingError.Context(codingPath: [key], debugDescription: "Value not found.")
            throw DecodingError.valueNotFound(type, context)
        }
        guard let value = Int(content) else {
            let context = DecodingError.Context(codingPath: [key], debugDescription: "Cannot read value as Int.")
            throw DecodingError.dataCorrupted(context)
        }
        return value
    }

    func decode(_ type: String.Type, forKey key: CodingKey) throws -> String {
        guard let group = self.content.substringGroups(matching: "^#\(key.stringValue):(.*)$").first else {
            let context = DecodingError.Context(codingPath: [], debugDescription: "Keyed String not found.")
            throw DecodingError.keyNotFound(key, context)
        }
        guard group.count == 2, case let value = group[1] else {
            let context = DecodingError.Context(codingPath: [key], debugDescription: "Value not found.")
            throw DecodingError.valueNotFound(type, context)
        }
        return String(value)
    }

    func decode(_ type: URL.Type, forKey key: CodingKey) throws -> URL {
        guard let group = self.content.substringGroups(matching: "^#\(key.stringValue):([\\w\\d\\-\\_/%:.?!&+=]+)$").first else {
            let context = DecodingError.Context(codingPath: [], debugDescription: "Keyed URL not found.")
            throw DecodingError.keyNotFound(key, context)
        }
        guard group.count == 2, case let content = group[1] else {
            let context = DecodingError.Context(codingPath: [key], debugDescription: "Value not found.")
            throw DecodingError.valueNotFound(type, context)
        }
        guard let value = URL(string: String(content)) else {
            let context = DecodingError.Context(codingPath: [key], debugDescription: "Cannot read value as URL.")
            throw DecodingError.dataCorrupted(context)
        }
        return value
    }

    func decode<T>(JSONValue type: T.Type, forKey key: CodingKey) throws -> T where T: Decodable {
        guard let group = self.content.substringGroups(matching: "^#\(key.stringValue):(.*)$").first else {
            let context = DecodingError.Context(codingPath: [], debugDescription: "Key not found.")
            throw DecodingError.keyNotFound(key, context)
        }
        guard group.count == 2, case let content = group[1] else {
            let context = DecodingError.Context(codingPath: [key], debugDescription: "Value not found.")
            throw DecodingError.valueNotFound(type, context)
        }
        
        guard let data = content.data(using: .utf8) else {
            let context = DecodingError.Context(codingPath: [key], debugDescription: "Cannot read value as JSON data.")
            throw DecodingError.dataCorrupted(context)
        }
        return try JSONDecoder().decode(type, from: data)
    }
}





// MARK: - Structures

struct M3UPlaylist {
    private var content: String

    init(data: Data) {
        self.content = String(data: data, encoding: .utf8)!
    }

    func audioPlaylistFilename() throws -> String {
        let playlists = content
            .substrings(matching: "^#EXT-X-MEDIA:.*$")
            .flatMap { $0.substrings(matching: "^.*(?::|,)TYPE=AUDIO(?:,.*)?$") }
            .flatMap { $0.substringGroups(matching: "(?::|,)URI=\"((?:\\w|-|_|\\.)+\\.m3u8)\"(?:,.*)?$") }
            .map { $0[1] }
        guard let audio = playlists.first else {
            throw GenericError(message: "Cannot find URI for audio stream file.")
        }
        return String(audio)
    }

    func videoPlaylistFilename() throws -> String {
        let playlists = content
            .substrings(matching: "^#EXT-X-STREAM-INF:.*\n[^#].*$")
            .flatMap { $0.substrings(matching: "^.*(?::|,)RESOLUTION=1920x1080(?:,.*)?\n.*$") }
            .flatMap { $0.substringGroups(matching: "^#.*\n((?:\\w|-|_|\\.)+\\.m3u8)$") }
            .map { $0[1] }
        guard let video = playlists.first else {
            throw GenericError(message: "Cannot find URI for video stream file.")
        }
        return String(video)
    }

    func mediaSegmentFilenames() throws -> [String] {
        let groups = self.content.substringGroups(matching: "^#EXT-X-MAP:URI=\"((?:\\w|-|_|\\.)+\\.m4s)\"$")
        guard let first = groups.first?[1] else {
            throw GenericError(message: "Cannot find the media stream's initial file.")
        }
        var filenames = [first]
        filenames.append(contentsOf: self.content.substrings(matching: "^[^#]((?:\\w|-|_|\\.)+\\.m4s)$"))
        return filenames.map(String.init(_:))
    }
}

struct SLSubtitleInfo: Codable {
    var name: String
    var language: String
    var url: URL
    var id: Int

    enum CodingKeys: String, CodingKey {
        case name
        case language
        case url = "webvtt_url"
        case id = "subtitles_id"
    }

    func downloadSubtitles(into directory: URL) async throws -> SLSubtitleInfo {
        let (data, _) = try await URLSession.shared.data(from: url)
        var content = String(data: data, encoding: .utf8)!
        content.range(of: "WEBVTT\n\n").map { content.removeSubrange($0) }
        let destination = directory
            .appendingPathComponent("subtitles_\(self.language)")
            .appendingPathExtension("vtt")
        try content.data(using: .utf8)!.write(to: destination)
        var subtitles = self
        subtitles.url = destination
        return subtitles
    }
}

struct SLSlide {
    var location: URL
    var timestamp: Int
}

struct SLSlideshow {
    var slides: [SLSlide]

    init(slides: [SLSlide]) {
        self.slides = slides
    }

    init(url: URL) async throws {
        let (data, _) = try await URLSession.shared.data(from: url)
        let document = try XMLDocument(data: data, options: .documentTidyXML)
        let names = try document.nodes(forXPath: "/videoContent/slide/slideName").compactMap(\.stringValue)
        let times = try document.nodes(forXPath: "/videoContent/slide/timeSec").compactMap(\.stringValue)
        let root = url.deletingLastPathComponent().deletingLastPathComponent()
            .appendingPathComponent("slides", isDirectory: true)
            .appendingPathComponent("big", isDirectory: true)
        self.slides = zip(names, times).map { name, time in
            let location = root.appendingPathComponent(name).appendingPathExtension("jpg")
            return SLSlide(location: location, timestamp: Int(time)!)
        }
    }

    func downloadSlides(into directory: URL) async throws -> SLSlideshow {
        await stdout.reportProgress(total: self.slides.count)
        let slides: [SLSlide] = try await self.slides.asyncMap { slide in
            let file = directory.appendingPathComponent(slide.location.lastPathComponent)
            await stdout.reportProgress()
            if slide.location.isFileURL {
                if slide.location != file {
                    try FileManager.default.copyItem(at: slide.location, to: file)
                }
            } else {
                let (data, _) = try await URLSession.shared.data(from: slide.location)
                try data.write(to: file, options: .atomic)
            }
            return SLSlide(location: file, timestamp: slide.timestamp)
        }
        return SLSlideshow(slides: slides)
    }

    func writeVideoConcatData(withRuntime runtime: Int, to url: URL) throws {
        guard !self.slides.isEmpty else {
            throw GenericError(message: "Cannot create an ffmpeg concat file for an empty presentation.")
        }
        guard !self.slides.map(\.location).map(\.isFileURL).contains(false) else {
            throw GenericError(message: "Cannot create an ffmpeg concat file with remote slides.")
        }
        let timestamps = self.slides.map(\.timestamp).appending(runtime)
        let durations = zip(timestamps.dropFirst(), timestamps).map(-)
        let content = zip(self.slides, durations)
            .map { (slide, duration) in "file '\(slide.location.path)'\nduration \(duration)" }
            .appending("file '\(self.slides.last!.location.path)'\n")
            .joined(separator: "\n")
        try content.data(using: .utf8)!.write(to: url, options: .atomic)
    }

    func writeSlideshowVideo(audio: URL, subtitles: [SLSubtitleInfo], to url: URL, tempdir: URL) throws {
        let duration = try ffmpeg.duration(of: audio)
        let concatFile = tempdir.appendingPathComponent("ffmpeg_concat.txt")
        try self.writeVideoConcatData(withRuntime: duration, to: concatFile)
        let rawVideoFile = tempdir.appendingPathComponent("raw_slides_video.mp4")
        try ffmpeg.writeVideoFromImages(using: concatFile, to: rawVideoFile)
        try ffmpeg.combineMediaFrom(video: rawVideoFile, audio: audio, subtitles: subtitles, to: url)
    }

    func writeSlideshowPDF(title: String, to url: URL) throws {
        guard url.isFileURL else {
            throw GenericError(message: "Cannot write to a non-file URL.")
        }
        let pdf = try PDFDocument(contentsOfImagesAt: self.slides.map(\.location))
        let pdfTitleKey = PDFDocumentWriteOption(rawValue: kCGPDFContextTitle as String)
        guard pdf.write(to: url, withOptions: [pdfTitleKey : title]) else {
            throw GenericError(message: "Cannot write PDF to \(url.path)")
        }
    }
}

struct SLPresentationInfo {
    var id: Int
    var title: String
    var service: String
    var videoID: String
    var servers: [String]
    var slidesXML: URL
    var subtitles: [SLSubtitleInfo]

    init(from decoder: M3UDecoder) throws {
        self.id = try decoder.decode(Int.self, forKey: CodingKeys.id)
        self.title = try decoder.decode(String.self, forKey: CodingKeys.title)
        self.service = try decoder.decode(String.self, forKey: CodingKeys.service)
        self.videoID = try decoder.decode(String.self, forKey: CodingKeys.videoID)
        self.servers = try decoder.decode(JSONValue: [String].self, forKey: CodingKeys.servers)
        self.slidesXML = try decoder.decode(URL.self, forKey: CodingKeys.slidesXML)
        self.subtitles = try decoder.decode(JSONValue: [SLSubtitleInfo].self, forKey: CodingKeys.subtitles)
    }

    enum CodingKeys: String, CodingKey {
        case id = "EXT-SL-PRESENTATION-ID"
        case title = "EXT-SL-PRESENTATION-TITLE"
        case service = "EXT-SL-VOD-VIDEO-SERVICE-NAME"
        case videoID = "EXT-SL-VOD-VIDEO-ID"
        case servers = "EXT-SL-VOD-VIDEO-SERVERS"
        case slidesXML = "EXT-SL-VOD-SLIDES-XML-URL"
        case subtitles = "EXT-SL-VOD-SUBTITLES"
    }

    init(id: String, token: String) async throws {
        let url = URL(string: "https://ben.slideslive.com/player/\(id)?player_token=\(token)")!
        let (data, _) = try await URLSession.shared.data(from: url)
        try self.init(from: M3UDecoder(data: data))
    }

    func fetchMediaStream() async throws -> SLMediaStream {
        try await SLMediaStream(server: self.servers.first!, id: self.videoID)
    }

    func fetchSlideshowInfo() async throws -> SLSlideshow {
        try await SLSlideshow(url: self.slidesXML)
    }
}

struct SLMediaStream {
    private var root: URL
    private var master: M3UPlaylist

    init(server: String, id: String) async throws {
        self.root = URL(string: "https://\(server)/\(id)")!
        let masterURL = self.root.appendingPathComponent("master.m3u8")
        self.master = try await SLMediaStream.fetchPlaylist(from: masterURL)
    }

    private static func fetchPlaylist(from url: URL) async throws -> M3UPlaylist {
        let (data, _) = try await URLSession.shared.data(from: url)
        return M3UPlaylist(data: data)
    }

    func downloadAudio(to url: URL) async throws {
        let filename = try self.master.audioPlaylistFilename()
        let playlist = try await SLMediaStream.fetchPlaylist(from: self.root.appendingPathComponent(filename))
        try await self.downloadStream(from: playlist, to: url)
    }

    func downloadVideo(to url: URL) async throws {
        let filename = try self.master.videoPlaylistFilename()
        let playlist = try await SLMediaStream.fetchPlaylist(from: self.root.appendingPathComponent(filename))
        try await self.downloadStream(from: playlist, to: url)
    }

    private func downloadStream(from playlist: M3UPlaylist, to url: URL) async throws {
        let urls = try playlist.mediaSegmentFilenames().map(self.root.appendingPathComponent(_:))
        await stdout.reportProgress(total: urls.count)
        let chunks = try await urls.asyncMap { url -> URL in
            let (url, _) = try await URLSession.shared.download(from: url)
            await stdout.reportProgress()
            return url
        }
        FileManager.default.createFile(atPath: url.path, contents: nil, attributes: nil)
        let handle = try FileHandle(forWritingTo: url)
        try chunks.forEach { chunk in
            try handle.write(contentsOf: try Data(contentsOf: chunk))
        }
        try handle.close()
    }
}

struct ffmpeg {
    enum Stream {
        case video(URL)
        case audio(URL)
        case subtitle(SLSubtitleInfo)

        var specifier: String {
            switch self {
            case .video(_): return "v"
            case .audio(_): return "a"
            case .subtitle(_): return "s"
            }
        }

        var url: URL {
            switch self {
            case let .video(url): return url
            case let .audio(url): return url
            case let .subtitle(info): return info.url
            }
        }
    }

    static func writeVideoFromImages(using concatFile: URL, to url: URL) throws {
        guard concatFile.isFileURL else {
            throw GenericError(message: "Cannot read ffmpeg info from non-file URL.")
        }
        guard url.isFileURL else {
            throw GenericError(message: "Cannot write to a non-file URL.")
        }
        let pipe = Pipe()
        let proc = Process()
        proc.launchPath = "/usr/local/bin/ffmpeg"
        proc.arguments = ["-y", "-nostdin",
                          "-f", "concat", "-safe", "0", "-i", concatFile.path,
                          "-vsync", "vfr", "-pix_fmt", "yuv420p",
                          "-vf", "crop=floor(iw/2)*2:floor(ih/2)*2", url.path]
        proc.standardError = pipe.fileHandleForWriting
        proc.launch()
        proc.waitUntilExit()
        defer {
            try? pipe.fileHandleForWriting.close()
            try? pipe.fileHandleForReading.close()
        }
        guard proc.terminationStatus == 0 else {
            throw ExitError(process: proc, stderr: pipe)
        }
    }

    static func combineMediaFrom(video: URL?, audio: URL?, subtitles: [SLSubtitleInfo], to url: URL) throws {
        guard url.isFileURL else {
            throw GenericError(message: "Cannot write to a non-file URL.")
        }
        let media = [video.map { Stream.video($0) }, audio.map { Stream.audio($0) }].compactMap { $0 }
        guard !media.isEmpty else {
            throw GenericError(message: "No URL provided for either video or audio media.")
        }
        let subtitles = subtitles.compactMap { info in
            languageCodes[info.language].map { lang in
                SLSubtitleInfo(name: info.name, language: lang, url: info.url, id: info.id)
            }
        }
        let input = media + subtitles.map { Stream.subtitle($0) }
        guard !input.map(\.url.isFileURL).contains(false) else {
            throw GenericError(message: "Cannot read ffmpeg media from non-file URL.")
        }
        var arguments = ["-y", "-nostdin"]
        arguments.append(contentsOf: zip(input, input.indices).flatMap { input, index in
            ["-i", input.url.path]
        })
        arguments.append(contentsOf: zip(input, input.indices).flatMap { input, index -> [String] in
            return ["-map", "\(index):\(input.specifier)"]
        })
        arguments.append(contentsOf: ["-c", "copy", "-c:s", "mov_text"])
        arguments.append(contentsOf: zip(subtitles, subtitles.indices).flatMap { info, index in
            [ "-metadata:s:s:\(index)", "language=\(info.language)" ]
        })
        arguments.append(url.path)
        let pipe = Pipe()
        let proc = Process()
        proc.launchPath = "/usr/local/bin/ffmpeg"
        proc.arguments = arguments
        proc.standardError = pipe.fileHandleForWriting
        proc.launch()
        proc.waitUntilExit()
        defer {
            try? pipe.fileHandleForWriting.close()
            try? pipe.fileHandleForReading.close()
        }
        guard proc.terminationStatus == 0 else {
            throw ExitError(process: proc, stderr: pipe)
        }
    }

    /// A mapping of alpha-2 language codes to alpha-3 language codes.
    static let languageCodes: [String : String] = [
        "aa" : "aar", "ab" : "abk", "ae" : "ave", "af" : "afr", "ak" : "aka", "am" : "amh", "an" : "arg", "ar" : "ara",
        "as" : "asm", "av" : "ava", "ay" : "aym", "az" : "aze", "ba" : "bak", "be" : "bel", "bg" : "bul", "bh" : "bih",
        "bi" : "bis", "bm" : "bam", "bn" : "ben", "bo" : "bod", "br" : "bre", "bs" : "bos", "ca" : "cat", "ce" : "che",
        "ch" : "cha", "co" : "cos", "cr" : "cre", "cs" : "ces", "cu" : "chu", "cv" : "chv", "cy" : "cym", "da" : "dan",
        "de" : "deu", "dv" : "div", "dz" : "dzo", "ee" : "ewe", "el" : "ell", "en" : "eng", "eo" : "epo", "es" : "spa",
        "et" : "est", "eu" : "eus", "fa" : "fas", "ff" : "ful", "fi" : "fin", "fj" : "fij", "fo" : "fao", "fr" : "fra",
        "fy" : "fry", "ga" : "gle", "gd" : "gla", "gl" : "glg", "gn" : "grn", "gu" : "guj", "gv" : "glv", "ha" : "hau",
        "he" : "heb", "hi" : "hin", "ho" : "hmo", "hr" : "hrv", "ht" : "hat", "hu" : "hun", "hy" : "hye", "hz" : "her",
        "ia" : "ina", "id" : "ind", "ie" : "ile", "ig" : "ibo", "ii" : "iii", "ik" : "ipk", "io" : "ido", "is" : "isl",
        "it" : "ita", "iu" : "iku", "ja" : "jpn", "jv" : "jav", "ka" : "kat", "kg" : "kon", "ki" : "kik", "kj" : "kua",
        "kk" : "kaz", "kl" : "kal", "km" : "khm", "kn" : "kan", "ko" : "kor", "kr" : "kau", "ks" : "kas", "ku" : "kur",
        "kv" : "kom", "kw" : "cor", "ky" : "kir", "la" : "lat", "lb" : "ltz", "lg" : "lug", "li" : "lim", "ln" : "lin",
        "lo" : "lao", "lt" : "lit", "lu" : "lub", "lv" : "lav", "mg" : "mlg", "mh" : "mah", "mi" : "mri", "mk" : "mkd",
        "ml" : "mal", "mn" : "mon", "mr" : "mar", "ms" : "msa", "mt" : "mlt", "my" : "mya", "na" : "nau", "nb" : "nob",
        "nd" : "nde", "ne" : "nep", "ng" : "ndo", "nl" : "nld", "nn" : "nno", "no" : "nor", "nr" : "nbl", "nv" : "nav",
        "ny" : "nya", "oc" : "oci", "oj" : "oji", "om" : "orm", "or" : "ori", "os" : "oss", "pa" : "pan", "pi" : "pli",
        "pl" : "pol", "ps" : "pus", "pt" : "por", "qu" : "que", "rm" : "roh", "rn" : "run", "ro" : "ron", "ru" : "rus",
        "rw" : "kin", "sa" : "san", "sc" : "srd", "sd" : "snd", "se" : "sme", "sg" : "sag", "si" : "sin", "sk" : "slo",
        "sl" : "slv", "sm" : "smo", "sn" : "sna", "so" : "som", "sq" : "alb", "sr" : "srp", "ss" : "ssw", "st" : "sot",
        "su" : "sun", "sv" : "swe", "sw" : "swa", "ta" : "tam", "te" : "tel", "tg" : "tgk", "th" : "tha", "ti" : "tir",
        "tk" : "tuk", "tl" : "tgl", "tn" : "tsn", "to" : "ton", "tr" : "tur", "ts" : "tso", "tt" : "tat", "tw" : "twi",
        "ty" : "tah", "ug" : "uig", "uk" : "ukr", "ur" : "urd", "uz" : "uzb", "ve" : "ven", "vi" : "vie", "vo" : "vol",
        "wa" : "wln", "wo" : "wol", "xh" : "xho", "yi" : "yid", "yo" : "yor", "za" : "zha", "zh" : "zho", "zu" : "zul"
    ]

    static func duration(of media: URL) throws -> Int {
        guard media.isFileURL else {
            throw GenericError(message: "Cannot read ffmpeg media from non-file URL.")
        }
        let pipe = Pipe()
        let proc = Process()
        proc.launchPath = "/usr/local/bin/ffprobe"
        proc.arguments = ["-i", media.path]
        proc.standardError = pipe.fileHandleForWriting
        proc.launch()
        proc.waitUntilExit()
        try? pipe.fileHandleForWriting.close()
        defer {
            try? pipe.fileHandleForReading.close()
        }
        guard proc.terminationStatus == 0 else {
            throw ExitError(process: proc, stderr: pipe)
        }
        guard let data = try pipe.fileHandleForReading.readToEnd() else {
            throw GenericError(message: "Cannot read media info for \(media.path)")
        }
        let content = String(data: data, encoding: .utf8)!
        let pattern = "^\\s*Duration:\\s+(\\d\\d):(\\d\\d):(\\d\\d).(\\d\\d)"
        guard let match = content.substringGroups(matching: pattern).first else {
            throw GenericError(message: "Cannot determine duration of \(media.path)")
        }
        let hour = Int(String(match[1]))!
        let min = Int(String(match[2]))!
        let sec = Int(String(match[3]))!
        let msec = Int(String(match[4]))!
        return (hour * 60 * 60) + (min * 60) + sec + ((msec + 99) / 100)
    }
}





// MARK: - Program

func fetchToken(id: String) async throws -> String {
    let url = URL(string: "https://slideslive.com/presentation/\(id)")!
    let (data, _) = try await URLSession.shared.data(from: url)
    let string = String(data: data, encoding: .utf8)!
    let regex = NSRegularExpression("<div id=\"player\".*\\sdata-player-token=\"((?:\\w|\\.|-|_)+)\"")
    let range = NSRange(string.startIndex..<string.endIndex, in: string)
    guard let match = regex.firstMatch(in: string, options: [], range: range) else {
        throw GenericError(message: "Cannot find the player token.")
    }
    let token = string[Range(match.range(at: 1), in: string)!]
    return String(token)
}

let currdir = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
let destdir = currdir.appendingPathComponent("Presentations", isDirectory: true)
let tempdir = try FileManager.default.url(for: .itemReplacementDirectory, in: .userDomainMask,
                                          appropriateFor: destdir, create: true)
if !FileManager.default.fileExists(atPath: destdir.path, isDirectory: nil) {
    try! FileManager.default.createDirectory(at: destdir, withIntermediateDirectories: false, attributes: nil)
}


let dateFormatter: DateFormatter = {
    let dateFormatter = DateFormatter()
    dateFormatter.dateFormat = "yyyy-MM-dd HH:mm"
    dateFormatter.timeZone = TimeZone(abbreviation: "GMT-7")
    dateFormatter.locale = Locale(identifier: "en_POSIX")
    return dateFormatter
}()

func cleanAttributesForItem(at url: URL, date: Date) throws {
    var attrs = try FileManager.default.attributesOfItem(atPath: url.path)
    attrs[FileAttributeKey.creationDate] = date
    attrs[.modificationDate] = date
    attrs[.extensionHidden] = true
    try FileManager.default.setAttributes(attrs, ofItemAtPath: url.path)
}

func downloadPresentation(id: String, to destdir: URL, filename: String, date: Date) async throws {
    let token = try await withStatus("Fetching token") { 
        try await fetchToken(id: id)
    }
    let prsinfo = try await withStatus("Fetching presentation info") {
        try await SLPresentationInfo(id: id, token: token) 
    }
    let slidesh = try await withStatus("Downloading slides") {
        try await prsinfo.fetchSlideshowInfo().downloadSlides(into: tempdir)
    }
    
    if !FileManager.default.fileExists(atPath: destdir.path) {
        try FileManager.default.createDirectory(at: destdir, withIntermediateDirectories: true, 
                                                attributes: [FileAttributeKey.creationDate : date])
    }
    let slideshow = destdir.appendingPathComponent("\(filename) (Slideshow)").appendingPathExtension("mp4")
    let video = destdir.appendingPathComponent("\(filename) (Presentation)").appendingPathExtension("mp4")
    let pdf = destdir.appendingPathComponent("\(filename) (Slides)").appendingPathExtension("pdf")
    
    try await withStatus("Saving presentation slides") {
        try slidesh.writeSlideshowPDF(title: prsinfo.title, to: pdf)
    }
    let vstream = try await withStatus("Fetching media stream") {
        try await prsinfo.fetchMediaStream()
    }
    let tempvideo: URL = try await withStatus("Downloading video stream") {
        let dest = tempdir.appendingPathComponent("video.mp4")
        try await vstream.downloadVideo(to: dest)
        return dest
    }
    let tempaudio: URL = try await withStatus("Downloading audio stream") {
        let dest = tempdir.appendingPathComponent("audio.mp4")
        try await vstream.downloadAudio(to: dest)
        return dest
    }
    let subtitles: [SLSubtitleInfo] = try await withStatus("Downloading subtitles") {
        await stdout.reportProgress(total: prsinfo.subtitles.count)
        return try await prsinfo.subtitles.asyncMap { info -> SLSubtitleInfo in
            await stdout.reportProgress()
            return try await info.downloadSubtitles(into: tempdir)
        }
    }
    try await withStatus("Saving presentation video") {
        try ffmpeg.combineMediaFrom(video: tempvideo, audio: tempaudio, subtitles: subtitles, to: video)
    }
    try await withStatus("Saving slideshow video") {
        try slidesh.writeSlideshowVideo(audio: tempaudio, subtitles: subtitles, to: slideshow, tempdir: tempdir)
    }

    try [destdir, slideshow, video, pdf].forEach { try cleanAttributesForItem(at: $0, date: date) }
}

guard FileManager.default.fileExists(atPath: "/usr/local/bin/ffmpeg") else {
    print("Cannot find ffmpeg utility at `/usr/local/bin/ffmpeg'.\n", to: &FileHandle.stderr)
    exit(1)
}

Task() {
    // https://www.pldi21.org/track_hopl.html
    let presentations = [
        (id: "38962614", date: "2021-06-20 06:00", filename: "Welcome to HOPL IV Conference"),
        (id: "38956885", date: "2021-06-20 06:15", filename: "Myths and Mythconceptions | What does it mean to be a programming language, anyhow?"),
        (id: "38956884", date: "2021-06-20 07:45", filename: "History of Coarrays and SPMD Parallelism in Fortran"),
        (id: "38956868", date: "2021-06-20 10:30", filename: "A History of MATLAB"),
        (id: "38956870", date: "2021-06-20 12:15", filename: "S, R and Data Science"),
        (id: "38956867", date: "2021-06-20 13:45", filename: "LabVIEW"),
        (id: "38956883", date: "2021-06-20 15:15", filename: "The Origins of Objective-C at PPI:Stepstone and its Evolution at NeXT"),
        (id: "38956875", date: "2021-06-20 16:45", filename: "JavaScript | The First 20 Years"),
        (id: "38956872", date: "2021-06-21 06:00", filename: "History of Logo"),
        (id: "38956877", date: "2021-06-21 07:45", filename: "A History of the Oz Multiparadigm Language"),
        (id: "38956869", date: "2021-06-21 10:30", filename: "Thriving in a Crowded and Changing World | C++ 2006-2020"),
        (id: "38956882", date: "2021-06-21 12:15", filename: "Origins of the D Programming Language"),
        (id: "38956874", date: "2021-06-21 13:45", filename: "A History of Clojure"),
        (id: "38956886", date: "2021-06-21 15:15", filename: "programmingLanguage as Language;"),
        (id: "38956879", date: "2021-06-21 16:45", filename: "The Evolution of Smalltalk from Smalltalk-72 through Squeak"),
        (id: "38956866", date: "2021-06-22 06:00", filename: "APL Since 1978"),
        (id: "38956871", date: "2021-06-22 07:45", filename: "Verilog HDL and its Ancestors and Descendants"),
        (id: "38956881", date: "2021-06-22 10:30", filename: "The History of Standard ML"),
        (id: "38956878", date: "2021-06-22 12:15", filename: "Evolution of Emacs Lisp"),
        (id: "38956880", date: "2021-06-22 13:45", filename: "The Early History of F#"),
        (id: "38956873", date: "2021-06-22 15:15", filename: "A History of the Groovy Programming Language"),
        (id: "38956876", date: "2021-06-22 16:45", filename: "Hygienic Macro Technology"),
    ]
    var failures: [(id: String, filename: String, error: Error)] = []
    for ((id, date, filename), index) in zip(presentations, presentations.indices) {
        do {
            await stdout.print("Downloading https://slideslive.com/presentation/\(id)")
            let destdir = destdir.appendingPathComponent(String(format: "%02d %@", index, filename), isDirectory: true)
            try await downloadPresentation(id: id, to: destdir, filename: filename, date: dateFormatter.date(from: date)!)
        } catch {
            await stderr.print(error)
            failures.append((id, filename, error))
        }
    }
    guard failures.isEmpty else {
        for failure in failures {
            await stderr.print("https://slideslive.com/presentation/\(failure.id) \(failure.filename)")
            await stderr.print("\t\(failure.error)".replacingOccurrences(of: "\n", with: "\n\t"))
        }
        exit(1)
    }
    exit(0)
}

RunLoop.main.run()

splice-video-presentation.sh

#!/bin/bash
#
# Copyright 2021 Steven Brunwasser
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

set -eo pipefail

# Splice slides between HOPL IV intro and Q&A session.
#
# $1 : The HOPL presentation video (with Q&A video)
# $2 : The slideshow video
# $3 : "hh:mm:ss" timestamp for the start of the Q&A session
# $4 : "YYYMMDDHHmmdd" timestamp for the presentation (optional)

PRESENTATION="$1"
SLIDESHOW="$2"
SLIDESHOW_START="00:00:12"
SLIDESHOW_END="$3"
ffmpeg -i "$PRESENTATION" -to 00:00:12 part1.mp4
ffmpeg -i "$SLIDESHOW" -vf scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:'(ow-iw)/2':'(oh-ih)/2',setsar=1 -to $SLIDESHOW_END -r 25 -c:a copy part2_resized.mp4
ffmpeg -i part2_resized.mp4 -ss $SLIDESHOW_START -r 25 part2.mp4
ffmpeg -ss $SLIDESHOW_END -i "$PRESENTATION" -c copy part3.mp4
ffmpeg -f concat -safe 0 -i <(for i in 1 2 3; do echo "file '$PWD/part$i.mp4'"; done) concat.mp4
ffmpeg -i concat.mp4 -i "$PRESENTATION" -map 0:a -map 0:v -map 1:s -c copy -c:s mov_text -metadata:s:s language=eng final.mp4
mv "$SLIDESHOW" "$SLIDESHOW.old"
mv final.mp4 "$SLIDESHOW"
rm part1.mp4 part2_resized.mp4 part2.mp4 part3.mp4 concat.mp4 "$SLIDESHOW.old"
if [ ${4:+set} ]; then touch -t $4 "$SLIDESHOW"; fi