Failing Avalonia LineBreaker tests

LineBreakerTests.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using Avalonia.Media.TextFormatting.Unicode;
using Avalonia.Utilities;
using Xunit;
using Xunit.Abstractions;

namespace Avalonia.Visuals.UnitTests.Media.TextFormatting
{
    public class LineBreakerTests
    {
        private readonly ITestOutputHelper output;

        public LineBreakerTests(ITestOutputHelper output) => this.output = output;


        [Fact]
        public void Should_Split_Text_By_Explicit_Breaks()
        {
            //ABC [0 3]
            //DEF\r[4 7]
            //\r[8]
            //Hello\r\n[9 15]
            const string text = "ABC DEF\r\rHELLO\r\n";

            var buffer = new ReadOnlySlice<char>(text.AsMemory());

            var lineBreaker = new LineBreakEnumerator(buffer);

            var current = 0;

            Assert.True(lineBreaker.MoveNext());

            var a = text.Substring(current, lineBreaker.Current.PositionMeasure - current + 1);

            Assert.Equal("ABC ", a);

            current += a.Length;

            Assert.True(lineBreaker.MoveNext());

            var b = text.Substring(current, lineBreaker.Current.PositionMeasure - current + 1);

            Assert.Equal("DEF\r", b);

            current += b.Length;

            Assert.True(lineBreaker.MoveNext());

            var c = text.Substring(current, lineBreaker.Current.PositionMeasure - current + 1);

            Assert.Equal("\r", c);

            current += c.Length;

            Assert.True(lineBreaker.MoveNext());

            var d = text.Substring(current, text.Length - current);

            Assert.Equal("HELLO\r\n", d);
        }

        [Fact]
        public void ICUTests() => Assert.True(this.ICUTestsImpl());

        // Contains over 7000 tests
        // https://www.unicode.org/Public/13.0.0/ucd/auxiliary/LineBreakTest.html
        public bool ICUTestsImpl()
        {
            this.output.WriteLine("Line Breaker Tests");
            this.output.WriteLine("------------------");

            // Read the test file
            string[] lines = File.ReadAllLines(Path.Combine(TestEnvironment.UnicodeTestDataFullPath, "LineBreakTest.txt"));

            // Process each line
            var tests = new List<Test>();
            for (int lineNumber = 1; lineNumber < lines.Length + 1; lineNumber++)
            {
                // Ignore deliberately skipped test?
                //if (SkipLines.Contains(lineNumber))
                //{
                //    continue;
                //}

                // Get the line, remove comments
                string line = lines[lineNumber - 1].Split('#')[0].Trim();

                // Ignore blank/comment only lines
                if (string.IsNullOrWhiteSpace(line))
                {
                    continue;
                }

                var codePoints = new List<int>();
                var breakPoints = new List<int>();

                // Parse the test
                int p = 0;
                while (p < line.Length)
                {
                    // Ignore white space
                    if (char.IsWhiteSpace(line[p]))
                    {
                        p++;
                        continue;
                    }

                    if (line[p] == '×')
                    {
                        p++;
                        continue;
                    }

                    if (line[p] == '÷')
                    {
                        breakPoints.Add(codePoints.Count);
                        p++;
                        continue;
                    }

                    int codePointPos = p;
                    while (p < line.Length && IsHexDigit(line[p]))
                    {
                        p++;
                    }

                    string codePointStr = line.Substring(codePointPos, p - codePointPos);
                    int codePoint = Convert.ToInt32(codePointStr, 16);
                    codePoints.Add(codePoint);
                }

                // Create test
                var test = new Test(lineNumber, codePoints.ToArray(), breakPoints.ToArray());
                tests.Add(test);
            }

            var foundBreaks = new List<int>();

            for (int testNumber = 0; testNumber < tests.Count; testNumber++)
            {
                Test t = tests[testNumber];

                foundBreaks.Clear();

                // Run the line breaker and build a list of break points
                var text = Encoding.UTF32.GetString(MemoryMarshal.Cast<int, byte>(t.CodePoints).ToArray());

                var enumerator = new LineBreakEnumerator(text.AsMemory());
                while (enumerator.MoveNext())
                {
                    foundBreaks.Add(enumerator.Current.PositionWrap);
                }

                // Check the same
                bool pass = true;
                if (foundBreaks.Count != t.BreakPoints.Length)
                {
                    pass = false;
                }
                else
                {
                    for (int i = 0; i < foundBreaks.Count; i++)
                    {
                        if (foundBreaks[i] != t.BreakPoints[i])
                        {
                            pass = false;
                        }
                    }
                }

                if (!pass)
                {
                    LineBreakClass[] classes = t.CodePoints.Select(x => UnicodeData.GetLineBreakClass(x)).ToArray();

                    this.output.WriteLine($"Failed test on line {t.LineNumber}");
                    this.output.WriteLine($"    Code Points: {string.Join(" ", t.CodePoints)}");
                    this.output.WriteLine($"Expected Breaks: {string.Join(" ", t.BreakPoints)}");
                    this.output.WriteLine($"  Actual Breaks: {string.Join(" ", foundBreaks)}");
                    this.output.WriteLine($"     Char Props: {string.Join(" ", classes)}");

                    return false;
                }
            }

            return true;
        }

        private static bool IsHexDigit(char ch) => char.IsDigit(ch) || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f');


        // The following test lines have been investigated and appear to be
        // expecting an incorrect result when compared to the default rules and
        // pair tables.
        private static readonly HashSet<int> SkipLines = new HashSet<int>()
        {
        };

        private readonly struct Test
        {
            public Test(int lineNumber, int[] codePoints, int[] breakPoints)
            {
                this.LineNumber = lineNumber;
                this.CodePoints = codePoints;
                this.BreakPoints = breakPoints;
            }

            public int LineNumber { get; }

            public int[] CodePoints { get; }

            public int[] BreakPoints { get; }
        }
    }

    internal static class TestEnvironment
    {
        private const string AvaloniaSolutionFileName = "Avalonia.sln";

        private const string UnicodeTestDataRelativePath = @"tests\TestFiles\UnicodeTestData\";

        private static readonly Lazy<string> SolutionDirectoryFullPathLazy = new Lazy<string>(GetSolutionDirectoryFullPathImpl);

        internal static string SolutionDirectoryFullPath => SolutionDirectoryFullPathLazy.Value;

        /// <summary>
        /// Gets the correct full path to the Unicode TestData directory.
        /// </summary>
        internal static string UnicodeTestDataFullPath => GetFullPath(UnicodeTestDataRelativePath);

        private static string GetSolutionDirectoryFullPathImpl()
        {
            string assemblyLocation = Path.GetDirectoryName(new Uri(typeof(TestEnvironment).GetTypeInfo().Assembly.CodeBase).LocalPath);

            var assemblyFile = new FileInfo(assemblyLocation);

            DirectoryInfo directory = assemblyFile.Directory;

            while (!directory.EnumerateFiles(AvaloniaSolutionFileName).Any())
            {
                try
                {
                    directory = directory.Parent;
                }
                catch (Exception ex)
                {
                    throw new Exception(
                        $"Unable to find Avalonia solution directory from {assemblyLocation} because of {ex.GetType().Name}!",
                        ex);
                }

                if (directory == null)
                {
                    throw new Exception($"Unable to find Avalonia solution directory from {assemblyLocation}!");
                }
            }

            return directory.FullName;
        }

        private static string GetFullPath(string relativePath) =>
            Path.Combine(SolutionDirectoryFullPath, relativePath)
            .Replace('\\', Path.DirectorySeparatorChar);
    }
}