Show intro chapterizer

Program.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SoundFingerprinting;
using SoundFingerprinting.Audio;
using SoundFingerprinting.Builder;
using SoundFingerprinting.Data;
using SoundFingerprinting.Emy.FFmpeg.Loader;
using SoundFingerprinting.InMemory;
using SoundFingerprinting.Query;

namespace IntroFingerprintingTest
{
    class Program
    {
        // This whole project works because of https://github.com/AddictedCS/soundfingerprinting and soundfingerprinting.Emy (add these nuget dependencies first!)
        // VLC tip: seek chapters using shift+P (previous) , shift+N (next)

        // ffmpeg: https://ffmpeg.org/download.html
        public static readonly string FfmpegPath = Path.GetFullPath(Path.Combine("FFmpeg", "bin", Environment.Is64BitProcess ? "x64" : "x86", "ffmpeg.exe"));

        // MKV tooling : https://www.fosshub.com/MKVToolNix.html
        public const string MkvToolPath = @"C:\Program Files\MKVToolNix\mkvmerge.exe";

        // Folder containing input AVI files to be "chapterized"
        public const string VideoFolder = @"input";

        public const string TimestampCsvName = @"timestamps.csv";
        public const string CsvCellDelimiter = ";";
        public const string CsvRowDelimiter = "\n";

        // The input filename must contain this string to be processed (for filtering by episodes). 
        // Change this string to the file extension to disable filtering
        public const string FilenameFilter = "S0";

        // How long is the intro, so that we can skip past this amount of time in seconds
        // public const int IntroductionPeriodToSkipSeconds = 52;
        
        public static Dictionary<string, Tuple<Timestamp, Timestamp>> FingerprintedIntroTimestamps = new ();

        private static IModelService modelService = new InMemoryModelService(); // store fingerprints in RAM
        
        // Need ffmpeg installed as described here to use the fingerprinting directly on video files. https://emysound.readme.io/reference/configuring-ffmpegaudioservice
        private static readonly IAudioService audioService = new SoundFingerprinting.Emy.FFmpegAudioService();

        enum RunMode 
        {
            FingerprintIntroTimestamps,
            AssembleMkvWithChapters,
            RunAll
        }

        static async Task Main(string[] args)
        {
            Console.WriteLine("Starting ChapterID");
            
            // The different stages are segmented for easier development, since it is undesirable to repeat earlier stages (e.g. fingerprinting) if a later stage (mkv generation) fails
            RunMode runMode = RunMode.RunAll;

            if (runMode == RunMode.FingerprintIntroTimestamps)
            {
                // Run fingerprinting and generate csv ONLY
                await RunFingerprintForAllVideos();
            }
            else if (runMode == RunMode.AssembleMkvWithChapters)
            {
                // Assemble mkv files from csv snapshot ONLY
                AddMkvChapters();
            }
            else if (runMode == RunMode.RunAll)
            {
                // Fingerprint AND build mkv files
                await RunFingerprintForAllVideos();
                AddMkvChapters();
            }
            else 
            {
                Console.WriteLine("No action specified");
            }
            
            // File.Delete(ReferenceEpisodeTrack);
            Console.WriteLine("Done");
            Console.ReadLine();
        }

        // Loads a previously stored set of timestamps from csv
        public static void LoadTimestampRecords() 
        {
            string[] csv = File.ReadAllText(TimestampCsvName).Trim().Split(CsvRowDelimiter);
            FingerprintedIntroTimestamps.Clear();
            foreach (string row in csv) 
            {
                string[] cells = row.Split(CsvCellDelimiter);
                if (cells.Length < 5) continue;
                var intro = new Timestamp(double.Parse(cells[1]), double.Parse(cells[2]));
                var credits = new Timestamp(double.Parse(cells[3]), double.Parse(cells[4]));
                FingerprintedIntroTimestamps.Add(cells[0], new Tuple<Timestamp, Timestamp>(intro, credits));
            }
        }

        // Loads file:timestamp records from csv, then creates "chapterized" mkvs
        public static void AddMkvChapters() 
        {
            LoadTimestampRecords();

            foreach (KeyValuePair<string, Tuple<Timestamp, Timestamp>> row in FingerprintedIntroTimestamps)
            {
                Console.WriteLine($"{row.Value.Item1.Start}-{row.Value.Item1.Start+row.Value.Item1.Duration} @ {row.Key}");

                // Convert the mkv, create the chapter data, then splice them together
                string mkvPath = RunFfmpegConvertMkv(Path.Combine(VideoFolder, row.Key));
                string chapterPath = CreateChapterFile(row.Value.Item1, row.Value.Item2);
                string newMkvPath = RunMkvSpliceChapter(mkvPath, chapterPath);

                // Cleanup: delete temporary mkv and chapter data
                File.Delete(chapterPath);
                File.Delete(mkvPath);

                Console.WriteLine($"Fixed {newMkvPath}");
            }
        }

        // Generates a chapter file containing 3 segments describing the intro's period, to be added into a mkv file
        public static string CreateChapterFile(Timestamp introTimestamp, Timestamp creditsTimestamp)
        {
            string chapterFilePath = $"{GetStartupPath()}{Path.DirectorySeparatorChar}{Guid.NewGuid()}.txt";
            TimeSpan startIntroTs = TimeSpan.FromSeconds(introTimestamp.Start);
            TimeSpan endIntroTs = TimeSpan.FromSeconds(introTimestamp.Start + introTimestamp.Duration);
            
            TimeSpan startCreditsTs = TimeSpan.FromSeconds(creditsTimestamp.Start);
            TimeSpan endCreditsTs = TimeSpan.FromSeconds(creditsTimestamp.Start + creditsTimestamp.Duration);

            string chapterNewline = "\n";
            string chapterTemplate = 
                $"CHAPTER01=00:00:00.000{chapterNewline}CHAPTER01NAME=Opening{chapterNewline}" +
                $"CHAPTER02=00:{startIntroTs.Minutes:D2}:{startIntroTs.Seconds:D2}.000{chapterNewline}CHAPTER02NAME=Introduction{chapterNewline}" +
                $"CHAPTER03=00:{endIntroTs.Minutes:D2}:{endIntroTs.Seconds:D2}.300{chapterNewline}CHAPTER03NAME=Content";

            if (creditsTimestamp.Start != -1)
            {
                chapterTemplate +=
                    $"{chapterNewline}CHAPTER04=00:{startCreditsTs.Minutes:D2}:{startCreditsTs.Seconds:D2}.000{chapterNewline}CHAPTER04NAME=Credits{chapterNewline}" +
                    $"CHAPTER05=00:{endCreditsTs.Minutes:D2}:{endCreditsTs.Seconds:D2}.300{chapterNewline}CHAPTER05NAME=AfterCredits";
            }

            File.WriteAllText(chapterFilePath, chapterTemplate);
            return chapterFilePath;
        }

        // Given a directory of avi files, fingerprint for the intro track and store its results in FingerprintedIntroTimestamps
        public static async Task RunFingerprintForAllVideos() 
        {
            Console.WriteLine("Loaded intro chunk");

            var files = Directory.GetFiles(VideoFolder);
            int firstAvi = -1, lastAvi = -1;

            for (int i = 0; i < files.Length; i++)
            {
                string targetFile = files[i];
                
                if ((Path.GetExtension(targetFile).ToLower() == ".avi" || Path.GetExtension(targetFile).ToLower() == ".mkv" || Path.GetExtension(targetFile).ToLower() == ".mp4"))
                {
                    if (firstAvi == -1)
                    {
                        firstAvi = i;
                        Console.WriteLine($"Matching against {targetFile}");
                        await StoreForLaterRetrieval(targetFile);
                        continue;
                    }

                    lastAvi = i;
                    
                    await RunFingerprint(targetFile);
                }
                else
                {
                    Console.WriteLine($"Skipping invalid file : {Path.GetFileName(targetFile)}");
                }
            }

            modelService = new InMemoryModelService();
            if (lastAvi != -1 && firstAvi != -1)
            {
                Console.WriteLine($"Matching against {files[lastAvi]}");
                await StoreForLaterRetrieval(files[lastAvi]);
                await RunFingerprint(files[firstAvi]);
            }

            SaveFingerprintResults();
        }

        public static void SaveFingerprintResults() 
        {
            StringBuilder csv = new StringBuilder();

            Console.WriteLine("=================================\nResults\n=================================\n");
            foreach (KeyValuePair<string, Tuple<Timestamp, Timestamp>> row in FingerprintedIntroTimestamps)
            {
                double roundedSecondIntro = Math.Round(row.Value.Item1.Start, 0);
                double roundedSecondDurationIntro = Math.Round(row.Value.Item1.Duration, 0);
                double roundedSecondCredits = Math.Round(row.Value.Item2.Start, 0);
                double roundedSecondDurationCredits = Math.Round(row.Value.Item2.Duration, 0);
                Console.WriteLine($"{roundedSecondIntro}-{roundedSecondIntro+roundedSecondDurationIntro} @ {row.Key}");
                csv.Append($"{row.Key}{CsvCellDelimiter}{roundedSecondIntro}{CsvCellDelimiter}{roundedSecondDurationIntro}{CsvCellDelimiter}{roundedSecondCredits}{CsvCellDelimiter}{roundedSecondDurationCredits}{CsvRowDelimiter}");
            }

            File.WriteAllText(TimestampCsvName, csv.ToString());
        }

        // Given an AVI file, find the timestamp of the intro, and store the results in FingerprintedIntroTimestamps
        public static async Task RunFingerprint(string targetFile)
        {
            // fingerprint for intro track, results stored in FingerprintedIntroTimestamps
            await RunFingerprintAsync(targetFile, targetFile);
        }

        // Gets the folder where the binary is executed in
        public static string GetStartupPath() 
        {
            return Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + Path.DirectorySeparatorChar;
        }

        // Converts the video *container* from AVI to MKV, as AVI does not seem to readily accept chapter data
        public static string RunFfmpegConvertMkv(string aviPath)
        {
            // this ffmpeg command does not re-encode the video, preventing quality degradation
            string mkvPath = $"{GetStartupPath()}{Path.DirectorySeparatorChar}{Path.GetFileNameWithoutExtension(aviPath)}.mkv";
            string ffmpegArgs = $"-fflags +genpts -i \"{aviPath}\" -c:v copy -c:a copy \"{mkvPath}\"";

            // Run ffmpeg, and wait for it to complete
            Process.Start(new ProcessStartInfo(FfmpegPath, ffmpegArgs)).WaitForExit();
            return mkvPath;
        }

        // Runs mkvmerge to splice a mkv file with a chapter file, and output a new mkv file
        public static string RunMkvSpliceChapter(string mkvPath, string chapterPath)
        {
            string outputMkvPath = $"{GetStartupPath()}{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}{Path.GetFileNameWithoutExtension(mkvPath)}_processed.mkv";
            if (File.Exists(outputMkvPath))
            {
                File.Delete(outputMkvPath);
            }
            string mkvArgs = $" --chapters \"{chapterPath}\" -o \"{outputMkvPath}\" \"{mkvPath}\"";

            // Run mkvmerge, and wait for it to complete
            Process.Start(new ProcessStartInfo(MkvToolPath, mkvArgs)).WaitForExit();
            return outputMkvPath;
        }

        // Checks a audio track for the presence and location of an intro track, then add it to the global results dictionary
        // Modified from the SoundFingerprinting example
        public static async Task RunFingerprintAsync(string pathToAudioFile, string originalAviPath) 
        {
            try
            {
                Console.WriteLine($"Matching {pathToAudioFile}");
                QueryResult queryResult = await QueryCommandBuilder.Instance
                    .BuildQueryCommand()
                    .From(pathToAudioFile)
                    .WithQueryConfig(config =>
                    {
                        config.AllowMultipleMatchesOfTheSameTrackInQuery = true;
                        config.PermittedGap = 3d;
                        return config;
                    })
                    .UsingServices(modelService, audioService)
                    .Query();

                string identifier = Path.GetFileName(originalAviPath);
                if (queryResult.BestMatch is null)
                {
                    Console.WriteLine("No matches");
                    FingerprintedIntroTimestamps.Add(identifier, new Tuple<Timestamp, Timestamp>(new Timestamp(-1, -1), new Timestamp(-1, -1)));
                }
                else
                {
                    var validResults = queryResult.ResultEntries.Where(r => r.Coverage.TrackCoverageLength > 15).ToList();
                    validResults.Sort((a, b) => (int) (a.QueryMatchStartsAt - b.QueryMatchStartsAt));
                    for (int i = 0; i < validResults.Count; i++)
                    {
                        var result = validResults[i];
                        Console.WriteLine($"Result({result.Confidence}) Start: {result.QueryMatchStartsAt}, Length: {result.Coverage.TrackCoverageLength}");
                    }
                
                    if (validResults.Count < 1)
                    {
                        Console.WriteLine("Rejected invalid result");
                    }
                    else if (validResults.Count == 1)
                    {
                        var match = validResults.First();
                        Console.WriteLine($"Fingerprint query: confidence {match.Confidence}, at {match.QueryMatchStartsAt}");
                        var intro = new Timestamp(match.QueryMatchStartsAt, match.Coverage.TrackCoverageLength);
                        var credits = new Timestamp(-1, -1);
                        FingerprintedIntroTimestamps.Add(identifier, new Tuple<Timestamp, Timestamp>(intro, credits));
                    }
                    else if (validResults.Count > 1)
                    {
                        var introMatch = validResults.First();
                        var creditsMatch = validResults.Last();
                        Console.WriteLine($"Fingerprint query: confidence {introMatch.Confidence}, intro at {introMatch.QueryMatchStartsAt}, credits at {creditsMatch.QueryMatchStartsAt}");
                        var intro = new Timestamp(introMatch.QueryMatchStartsAt, introMatch.Coverage.TrackCoverageLength);
                        var credits = new Timestamp(creditsMatch.QueryMatchStartsAt, creditsMatch.Coverage.TrackCoverageLength);
                        FingerprintedIntroTimestamps.Add(identifier, new Tuple<Timestamp, Timestamp>(intro, credits));
                    }
                    else
                    {
                        Console.WriteLine("Rejected invalid result");
                    }
                }
            }
            catch (Exception e)
            {
                Console.WriteLine("[RunFingerprintAsync] " + e);
            }
        }

        // Adds a audio track to be recognized, in this case, the video's introduction track
        // Mostly copied from the SoundFingerprinting example
        public static async Task StoreForLaterRetrieval(string pathToAudioFile)
        {
            // Anything here is fine since we only want to know *when* the track is playing
            var track = new TrackInfo("INTRO_ID", "INTRO_TITLE", "INTRO_ARTIST");

            // Create fingerprint from the wav file
            var hashedFingerprints = await FingerprintCommandBuilder.Instance
                .BuildFingerprintCommand()
                .From(pathToAudioFile)
                .UsingServices(audioService)
                .Hash();

            // Store hashes in the database for later retrieval
            modelService.Insert(track, hashedFingerprints);
        }
    }

    public struct Timestamp
    {
        public double Start;
        public double Duration;

        public Timestamp(double start, double duration)
        {
            Start = start;
            Duration = duration;
        }
    }
}