diff --git a/LibLouis.NET.Test/BrailleSpec.cs b/LibLouis.NET.Test/BrailleSpec.cs new file mode 100644 index 0000000..a3e944e --- /dev/null +++ b/LibLouis.NET.Test/BrailleSpec.cs @@ -0,0 +1,398 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +using YamlDotNet.Core; +using YamlDotNet.Core.Events; + +namespace LibLouis.NET.Test; + +/// +/// One translation case from an upstream braille spec. +/// +public sealed record BrailleSpecCase( + string SpecFile, + int Line, + string TableQuery, + string? AssertMatch, + string DisplayTable, + string Input, + string Expected, + TestDirection Direction, + bool ExpectedToFail) +{ + public override string ToString() => + $"{SpecFile}:{Line} {Direction} {Describe(Input)} -> {Describe(Expected)}"; + + // Braille output is mostly U+28xx, which is unreadable in a test runner's output, so show the + // code points for anything outside printable ASCII. + private static string Describe(string value) => + value.All(c => c is >= ' ' and <= '~') + ? $"\"{value}\"" + : string.Concat(value.Select(c => c is >= ' ' and <= '~' ? c.ToString() : $"\\u{(int)c:X4}")); +} + +public enum TestDirection +{ + Forward, + Backward, +} + +/// +/// How a spec entry's two values are used. The distinction matters: the backward leg of +/// bothDirections swaps input and expected (lou_checkyaml.c:900-902), while an explicit +/// backward testmode does not (lou_checkyaml.c:892-895). +/// +public enum TestMode +{ + Forward, + Backward, + BothDirections, +} + +/// +/// Reads liblouis braille spec files. +/// +/// +/// These files are not YAML mappings and cannot be deserialised. A single document repeats +/// table, flags and tests at the same level, which duplicate key handling +/// would collapse or reject. lou_checkyaml treats the file as an event stream where each key +/// mutates parser state, and tests executes against whatever is current +/// (tools/lou_checkyaml.c:1087-1139), so this reader does the same over YamlDotNet's IParser. +/// +/// Consecutive table keys accumulate rather than replace: the following tests block +/// runs once per accumulated table. flags persists until the next flags. +/// +/// Only the constructs the Danish specs actually use are supported. Anything else throws rather +/// than being skipped, so a spec using a feature this reader does not model fails loudly instead +/// of silently testing less than it appears to. +/// +public static class BrailleSpecReader +{ + public static IReadOnlyList Read(string path) + { + string specFile = Path.GetFileName(path); + var cases = new List(); + + using var reader = new StreamReader(path); + var parser = new Parser(reader); + + parser.Consume(); + parser.Consume(); + parser.Consume(); + + string displayTable = string.Empty; + var tables = new List<(string Query, string? AssertMatch)>(); + TestMode mode = TestMode.Forward; + + // Consecutive table keys accumulate, but the first one after a tests block starts a fresh + // set rather than adding to the one just used. + bool tablesUsed = false; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + + switch (key) + { + case "display": + displayTable = ReadTableValue(parser).Query; + break; + + case "table": + if (tablesUsed) + { + tables.Clear(); + tablesUsed = false; + } + + tables.Add(ReadTableValue(parser)); + break; + + case "flags": + mode = ReadFlags(parser); + break; + + case "tests": + ReadTests(parser, specFile, tables, displayTable, mode, cases); + tablesUsed = true; + break; + + default: + throw new NotSupportedException( + $"{specFile}: unsupported top level key '{key}'. This reader models only the " + + "constructs the Danish specs use; see lou_checkyaml.c for the full format."); + } + + } + + return cases; + } + + /// + /// A table value is either a file name or a query mapping. Queries are passed to lou_findTable + /// as "key:value key:value"; __assert-match is a harness directive, not part of the query. + /// + private static (string Query, string? AssertMatch) ReadTableValue(IParser parser) + { + if (parser.Current is Scalar scalar) + { + parser.MoveNext(); + return (scalar.Value, null); + } + + parser.Consume(); + + var terms = new List(); + string? assertMatch = null; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + string value = parser.Consume().Value; + + if (key == "__assert-match") + { + assertMatch = value; + } + else + { + terms.Add($"{key}:{value}"); + } + } + + parser.Consume(); + + return (string.Join(' ', terms), assertMatch); + } + + private static TestMode ReadFlags(IParser parser) + { + parser.Consume(); + + TestMode mode = TestMode.Forward; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + string value = parser.Consume().Value; + + if (key != "testmode") + { + throw new NotSupportedException($"unsupported flag '{key}'"); + } + + mode = ParseTestMode(value); + } + + parser.Consume(); + + return mode; + } + + private static TestMode ParseTestMode(string value) => value switch + { + "forward" => TestMode.Forward, + "backward" => TestMode.Backward, + "bothDirections" => TestMode.BothDirections, + _ => throw new NotSupportedException($"unsupported testmode '{value}'"), + }; + + private static void ReadTests( + IParser parser, + string specFile, + List<(string Query, string? AssertMatch)> tables, + string displayTable, + TestMode mode, + List cases) + { + parser.Consume(); + + while (parser.Current is not SequenceEnd) + { + SequenceStart entryStart = parser.Consume(); + int line = (int)entryStart.Start.Line; + + string input = Unescape(parser.Consume().Value); + string expected = Unescape(parser.Consume().Value); + + var xfail = XFail.None; + TestMode entryMode = mode; + bool skip = false; + + if (parser.Current is MappingStart) + { + (xfail, entryMode, skip) = ReadTestOptions(parser, mode); + } + + parser.Consume(); + + if (skip) + { + continue; + } + + foreach ((string query, string? assertMatch) in tables) + { + // Forward compares translate(input) with expected. An explicit backward testmode + // means the entry is already written braille-first, so it is not swapped. The + // backward leg of bothDirections is: the expected braille is the input, and the + // original text is what back translation should produce. + if (entryMode is TestMode.Forward or TestMode.BothDirections) + { + cases.Add(new BrailleSpecCase( + specFile, line, query, assertMatch, displayTable, + input, expected, TestDirection.Forward, xfail.HasFlag(XFail.Forward))); + } + + if (entryMode == TestMode.Backward) + { + cases.Add(new BrailleSpecCase( + specFile, line, query, assertMatch, displayTable, + input, expected, TestDirection.Backward, xfail.HasFlag(XFail.Backward))); + } + else if (entryMode == TestMode.BothDirections) + { + cases.Add(new BrailleSpecCase( + specFile, line, query, assertMatch, displayTable, + expected, input, TestDirection.Backward, xfail.HasFlag(XFail.Backward))); + } + } + } + + parser.Consume(); + } + + [Flags] + private enum XFail + { + None = 0, + Forward = 1, + Backward = 2, + Both = Forward | Backward, + } + + private static (XFail XFail, TestMode Mode, bool Skip) ReadTestOptions( + IParser parser, TestMode mode) + { + parser.Consume(); + + var xfail = XFail.None; + bool skip = false; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + + switch (key) + { + case "xfail": + xfail = ReadXFail(parser); + break; + + case "testmode": + mode = ParseTestMode(parser.Consume().Value); + break; + + // Emphasis is applied through typeform, which this prototype does not drive yet. + // Skip the value so the rest of the file still parses, and drop the case: silently + // running it without the typeform would compare against the wrong expectation. + case "typeform": + parser.SkipThisAndNestedEvents(); + skip = true; + break; + + default: + throw new NotSupportedException($"unsupported test option '{key}'"); + } + } + + parser.Consume(); + + return (xfail, mode, skip); + } + + /// + /// xfail is either a scalar, where only "false" and "off" are falsy + /// (tools/lou_checkyaml.c:379-389), or a mapping naming the failing directions. + /// + private static XFail ReadXFail(IParser parser) + { + if (parser.Current is Scalar scalar) + { + parser.MoveNext(); + return scalar.Value is "false" or "off" ? XFail.None : XFail.Both; + } + + parser.Consume(); + + var xfail = XFail.None; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + string value = parser.Consume().Value; + bool set = value is not ("false" or "off"); + + if (set) + { + xfail |= key switch + { + "forward" => XFail.Forward, + "backward" => XFail.Backward, + _ => throw new NotSupportedException($"unsupported xfail direction '{key}'"), + }; + } + } + + parser.Consume(); + + return xfail; + } + + /// + /// The specs use single quoted scalars, where YAML performs no escape processing at all, and + /// rely on liblouis to interpret the escapes itself. Only the forms the Danish specs actually + /// use are handled: \xNNNN and \uNNNN code points, and \\ for a literal backslash. + /// Without the backslash case, 'at\\bliver' parses as two backslashes and translates to two + /// cells where upstream expects one. + /// + private static string Unescape(string value) + { + if (!value.Contains('\\', StringComparison.Ordinal)) + { + return value; + } + + var builder = new StringBuilder(value.Length); + + for (int i = 0; i < value.Length; i++) + { + if (value[i] == '\\' && i + 1 < value.Length) + { + if (value[i + 1] is 'x' or 'y' or 'u' && i + 5 < value.Length && + ushort.TryParse( + value.AsSpan(i + 2, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out ushort code)) + { + builder.Append((char)code); + i += 5; + continue; + } + + if (value[i + 1] is '\\' or '"') + { + builder.Append(value[i + 1]); + i++; + continue; + } + } + + builder.Append(value[i]); + } + + return builder.ToString(); + } +} diff --git a/LibLouis.NET.Test/BrailleSpecTests.cs b/LibLouis.NET.Test/BrailleSpecTests.cs new file mode 100644 index 0000000..3a8dc95 --- /dev/null +++ b/LibLouis.NET.Test/BrailleSpecTests.cs @@ -0,0 +1,194 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +using Xunit; + +namespace LibLouis.NET.Test; + +/// +/// Runs the upstream braille specs for Danish through the managed wrapper. +/// +/// +/// These are liblouis's own expectations, so they check the wrapper end to end against thousands of +/// cases nobody here had to invent: not just that a translation succeeds, but that it produces the +/// characters upstream says it should, in both directions. +/// +/// The specs live in braille-specs/ and are copied verbatim from +/// upstream/liblouis-<version>/tests/braille-specs/. Re-copy them when the upstream version is +/// bumped; the diff is the set of expectations that changed. +/// +/// One test per spec file rather than per case. Ten thousand xunit cases makes discovery slow and +/// buries a real regression in an unreadable log; a single failure listing every mismatch is more +/// use than ten thousand separate red entries. +/// +public class BrailleSpecTests +{ + private static readonly string SpecDirectory = + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "braille-specs"); + + // tables/ is the upstream set. Nota's own tables live in nota-tables/ and no longer shadow it, + // so the specs can be checked against the tables they were actually written for. + private static readonly string TableDirectory = + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables"); + + public static TheoryData SpecFiles() + { + var data = new TheoryData(); + + foreach (string path in Directory.EnumerateFiles(SpecDirectory, "*.yaml").OrderBy(f => f, StringComparer.Ordinal)) + { + data.Add(Path.GetFileName(path)); + } + + return data; + } + + [Theory] + [MemberData(nameof(SpecFiles))] + public void MatchesUpstreamExpectations(string specFile) + { + IReadOnlyList cases = BrailleSpecReader.Read(Path.Combine(SpecDirectory, specFile)); + + IndexUpstreamTables(); + + var mismatches = new List(); + var unexpectedPasses = new List(); + int checkedCount = 0; + + foreach (BrailleSpecCase testCase in cases) + { + + string table = ResolveTable(testCase, TableCache.Value); + string? actual = Run(testCase, table); + bool matched = actual == testCase.Expected; + checkedCount++; + + if (testCase.ExpectedToFail) + { + // Upstream reports an unexpected pass as a warning rather than an error: the case + // is known-broken, and it passing usually means the expectation moved. + if (matched) + { + unexpectedPasses.Add(testCase.ToString()); + } + } + else if (!matched) + { + mismatches.Add($"{testCase}\n actual: {Describe(actual)}"); + } + } + + + Assert.True( + mismatches.Count == 0, + $"{specFile}: {mismatches.Count} of {checkedCount} cases did not match upstream " + + $"({unexpectedPasses.Count} xfail cases passed unexpectedly).\n " + + string.Join("\n ", mismatches.Take(25)) + + (mismatches.Count > 25 ? $"\n ... and {mismatches.Count - 25} more" : string.Empty)); + } + + private static string? Run(BrailleSpecCase testCase, string table) + { + string[] tables = [ResolveDisplayTable(testCase.DisplayTable), table]; + int outputLength = Math.Max(testCase.Input.Length, testCase.Expected.Length) * 4; + + try + { + return testCase.Direction == TestDirection.Forward + ? LibLouis.Instance.Translate(tables, testCase.Input, outputLength, null, null, TranslationMode.Regular) + : LibLouis.Instance.BackTranslate(tables, testCase.Input, outputLength, null, null, TranslationMode.Regular); + } + catch (LibLouisException ex) + { + // A translation that fails outright is a mismatch, not an error: upstream marks some of + // these xfail, and letting it throw would stop the whole file at the first one. The + // message is carried into the comparison so a failure says why, rather than only that + // nothing came back. + return $""; + } + } + + // Every query from every spec is resolved once, up front, before any translation runs. + // lou_findTable's return value is freed by the marshaller with the wrong allocator (P/Invoke + // audit item 3), so interleaving these calls with translations corrupts the native heap. + private static readonly Lazy> TableCache = new(() => + { + LibLouis.Instance.IndexTables( + Directory.EnumerateFiles(TableDirectory) + .Where(f => Path.GetExtension(f) is ".ctb" or ".utb" or ".uti" or ".dis" or ".cti" or ".dic")); + + var resolved = new Dictionary(StringComparer.Ordinal); + + foreach (string query in Directory.EnumerateFiles(SpecDirectory, "*.yaml") + .SelectMany(BrailleSpecReader.Read) + .Select(c => c.TableQuery) + .Distinct(StringComparer.Ordinal)) + { + resolved[query] = LibLouis.Instance.FindTable(query) ?? string.Empty; + } + + return resolved; + }); + + private static void IndexUpstreamTables() => _ = TableCache.Value; + + /// + /// Resolving a table query is what lou_findTable does, and the specs assert which file a query + /// should select, so this covers table resolution as well as translation. + /// + private static string ResolveTable(BrailleSpecCase testCase, Dictionary cache) + { + if (!cache.TryGetValue(testCase.TableQuery, out string? resolved)) + { + resolved = LibLouis.Instance.FindTable(testCase.TableQuery) ?? string.Empty; + cache[testCase.TableQuery] = resolved; + } + + Assert.False( + string.IsNullOrEmpty(resolved), + $"No table matched the query '{testCase.TableQuery}' from {testCase.SpecFile}:{testCase.Line}"); + + if (testCase.AssertMatch is not null) + { + Assert.True( + string.Equals(Path.GetFileName(resolved), testCase.AssertMatch, StringComparison.Ordinal), + $"Query '{testCase.TableQuery}' resolved to {Path.GetFileName(resolved)}, " + + $"but {testCase.SpecFile}:{testCase.Line} asserts {testCase.AssertMatch}"); + } + + return resolved; + } + + /// + /// A spec's display table is usually a file name, but it can also be an inline table written as + /// a YAML block scalar, for instance to include the standard one and then override a character. + /// liblouis only takes paths, so the inline form is written out next to the tables, where the + /// includes inside it resolve. + /// + private static string ResolveDisplayTable(string display) + { + // A file name never contains a newline, so this distinguishes the two forms. + if (!display.Contains('\n', StringComparison.Ordinal)) + { + return Path.Combine(TableDirectory, display); + } + + string name = $"inline-{Convert.ToHexString(System.Security.Cryptography.MD5.HashData(System.Text.Encoding.UTF8.GetBytes(display)))[..8]}.dis"; + string path = Path.Combine(TableDirectory, name); + + if (!File.Exists(path)) + { + File.WriteAllText(path, display); + } + + return path; + } + + private static string Describe(string? value) => + value is null + ? "" + : string.Concat(value.Select(c => c is >= ' ' and <= '~' ? c.ToString() : $"\\u{(int)c:X4}")); +} diff --git a/LibLouis.NET.Test/LibLouis.NET.Test.csproj b/LibLouis.NET.Test/LibLouis.NET.Test.csproj index 40fc277..1e8ca27 100644 --- a/LibLouis.NET.Test/LibLouis.NET.Test.csproj +++ b/LibLouis.NET.Test/LibLouis.NET.Test.csproj @@ -17,6 +17,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -29,6 +30,15 @@ + + + PreserveNewest + +