From 9489d363978aa17307bb8901516d8e6930efa272 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Sun, 1 Feb 2026 13:11:16 -0800 Subject: [PATCH 01/41] Add draft for benchmarking tool --- Harp.Toolkit/Benchmark/BenchmarkCommand.cs | 152 +++++++++++++++++ Harp.Toolkit/Benchmark/HarpTestAttribute.cs | 7 + Harp.Toolkit/Benchmark/HtmlReportGenerator.cs | 21 +++ Harp.Toolkit/Benchmark/Report.cs | 8 + Harp.Toolkit/Benchmark/ReportTemplate.cshtml | 116 +++++++++++++ Harp.Toolkit/Benchmark/Result.cs | 161 ++++++++++++++++++ Harp.Toolkit/Benchmark/Runner.cs | 53 ++++++ Harp.Toolkit/Benchmark/Suite.cs | 71 ++++++++ .../Benchmark/Suites/RoundTripTestSuite.cs | 42 +++++ .../Benchmark/Suites/TimestampSecondSuite.cs | 24 +++ Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs | 21 +++ src/Harp.Toolkit/Program.cs | 1 + 12 files changed, 677 insertions(+) create mode 100644 Harp.Toolkit/Benchmark/BenchmarkCommand.cs create mode 100644 Harp.Toolkit/Benchmark/HarpTestAttribute.cs create mode 100644 Harp.Toolkit/Benchmark/HtmlReportGenerator.cs create mode 100644 Harp.Toolkit/Benchmark/Report.cs create mode 100644 Harp.Toolkit/Benchmark/ReportTemplate.cshtml create mode 100644 Harp.Toolkit/Benchmark/Result.cs create mode 100644 Harp.Toolkit/Benchmark/Runner.cs create mode 100644 Harp.Toolkit/Benchmark/Suite.cs create mode 100644 Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs create mode 100644 Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs create mode 100644 Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs diff --git a/Harp.Toolkit/Benchmark/BenchmarkCommand.cs b/Harp.Toolkit/Benchmark/BenchmarkCommand.cs new file mode 100644 index 0000000..f6754ef --- /dev/null +++ b/Harp.Toolkit/Benchmark/BenchmarkCommand.cs @@ -0,0 +1,152 @@ +using System.CommandLine; +using Spectre.Console; + +namespace Harp.Toolkit; +public class BenchmarkCommand : Command +{ + public BenchmarkCommand() + : base("benchmark", "Run benchmark tests on the device.") + { + PortNameOption portNameOption = new(); + Option fileOption = new("--report") + { + Description = "Path to the HTML report generated after running tests.", + Required = false, + }; + + Option verboseOption = new("--verbose") + { + Description = "Show detailed results for each test.", + Required = false, + }; + Options.Add(portNameOption); + Options.Add(fileOption); + Options.Add(verboseOption); + SetAction(parsedResult => + { + string portName = parsedResult.GetRequiredValue(portNameOption); + FileInfo? reportFile = parsedResult.GetValue(fileOption); + bool verbose = parsedResult.GetValue(verboseOption); + return RunBenchmarks(portName, reportFile, verbose, CancellationToken.None); + }); + } + + static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verbose, CancellationToken cancellationToken) + { + AnsiConsole.MarkupLine($"Running tests on [bold]{portName}[/]..."); + + var runner = new CoreRunner(); + var report = new Report + { + DeviceName = $"Harp Device ({portName})", + RunDate = DateTime.Now + }; + + await AnsiConsole.Progress() + .StartAsync(async ctx => + { + var task = ctx.AddTask("[green]Running tests...[/]", true, runner.TestCount); + + await foreach (var (suite, result) in runner.RunAllAsync(portName, cancellationToken)) + { + task.Increment(1); + AnsiConsole.MarkupLine($"[grey]{suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}"); + + var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name); + if (suiteResult == null) + { + suiteResult = new SuiteResult + { + Name = suite.GetType().Name, + Description = suite.Description + }; + report.Suites.Add(suiteResult); + } + suiteResult.Results.Add(result); + } + }); + + if (verbose) + { + AnsiConsole.WriteLine(); + AnsiConsole.Write(new Rule("[yellow]Detailed Results[/]")); + foreach (var suite in report.Suites) + { + AnsiConsole.MarkupLine($"[bold underline]{suite.Name}[/]"); + AnsiConsole.MarkupLine($"[dim]{suite.Description}[/]"); + + var table = new Table(); + table.AddColumn("Test Case"); + table.AddColumn("Status"); + table.AddColumn("Details"); + table.AddColumn("Message"); + + foreach (var test in suite.Results) + { + string details = ""; + string message = test.Result.Message ?? ""; + + if (test.Result is NumericBenchmarkResult bsr) + { + details = $"Mean: {bsr.Summary.Mean:F4}\nMedian: {bsr.Summary.Median:F4}\nStdDev: {bsr.Summary.StdDev:F4}\nMin: {bsr.Summary.Min:F4}\nMax: {bsr.Summary.Max:F4}\nPercentiles: 99th={bsr.Summary.Percentile99:F4}, 01th={bsr.Summary.Percentile01:F4}"; + } + else if (test.Result is ErrorResult er) + { + details = $"{er.Exception.GetType().Name}"; + } + else + { + var valProp = test.Result?.GetType().GetProperty("Value"); + if (valProp != null) + { + var val = valProp.GetValue(test.Result); + details = val?.ToString() ?? ""; + } + } + + table.AddRow( + new Markup($"[bold]{test.Name}[/]\n[dim]{test.Description}[/]"), + new Markup(GetResultMarkup(test.Result)), + new Markup(details), + new Markup(message) + ); + } + AnsiConsole.Write(table); + AnsiConsole.WriteLine(); + } + } + + if (reportFile != null) + { + AnsiConsole.Markup("Generating HTML report..."); + string html = await HtmlReportGenerator.GenerateAsync(report); + string fileName = reportFile?.FullName ?? $"TestReport_{DateTime.Now:yyyyMMdd_HHmmss}.html"; + await File.WriteAllTextAsync(fileName, html, cancellationToken); + AnsiConsole.MarkupLine($"[green]Done![/] Report generated: [link]{fileName}[/]"); + } + } + + static string GetResultMarkup(IResult result) + { + return result.Status switch + { + Status.Passed => "[green]Passed[/]", + Status.Failed => "[red]Failed[/]", + Status.Error => "[red]Error[/]", + Status.Skipped => "[yellow]Skipped[/]", + _ => $"[white]{result.Status}[/]" + }; + } + + class CoreRunner : Runner + { + public CoreRunner() : base() + { + AddSuite(new WhoAmISuite()); + AddSuite(new RoundTripTestSuite()); + AddSuite(new TimestampSecondsSuite()); + } + } +} + + diff --git a/Harp.Toolkit/Benchmark/HarpTestAttribute.cs b/Harp.Toolkit/Benchmark/HarpTestAttribute.cs new file mode 100644 index 0000000..f0bf7cb --- /dev/null +++ b/Harp.Toolkit/Benchmark/HarpTestAttribute.cs @@ -0,0 +1,7 @@ +namespace Harp.Toolkit; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public class HarpTestAttribute : Attribute +{ + public string? Description { get; set; } +} diff --git a/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs b/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs new file mode 100644 index 0000000..ede8e25 --- /dev/null +++ b/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs @@ -0,0 +1,21 @@ +using System.Reflection; +using RazorLight; + +namespace Harp.Toolkit; + +public static class HtmlReportGenerator +{ + public static async Task GenerateAsync(Report report) + { + var engine = new RazorLightEngineBuilder() + .UseFileSystemProject(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)) + .UseMemoryCachingProvider() + .Build(); + + // The template is copied to the output directory under Reporting/ReportTemplate.cshtml + // RazorLight expects the path relative to the project root (which we set to the assembly location) + string templatePath = Path.Combine("Benchmark", "ReportTemplate.cshtml"); + + return await engine.CompileRenderAsync(templatePath, report); + } +} diff --git a/Harp.Toolkit/Benchmark/Report.cs b/Harp.Toolkit/Benchmark/Report.cs new file mode 100644 index 0000000..cc716f2 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Report.cs @@ -0,0 +1,8 @@ +namespace Harp.Toolkit; + +public class Report +{ + public string DeviceName { get; set; } = "Unknown Device"; + public DateTime RunDate { get; set; } = DateTime.Now; + public List Suites { get; set; } = new(); +} diff --git a/Harp.Toolkit/Benchmark/ReportTemplate.cshtml b/Harp.Toolkit/Benchmark/ReportTemplate.cshtml new file mode 100644 index 0000000..9cdfbac --- /dev/null +++ b/Harp.Toolkit/Benchmark/ReportTemplate.cshtml @@ -0,0 +1,116 @@ +@using Harp.Toolkit +@model Harp.Toolkit.Report + + + + + + + Test Report - @Model.DeviceName + + + + +
+
+
+

@Model.DeviceName

+

Test Execution Report • @Model.RunDate.ToString("MMMM dd, yyyy HH:mm:ss")

+
+
+ Harp Toolkit Test +
+
+ + @foreach (var suite in Model.Suites) + { +
+
+

@suite.Name

+

@suite.Description

+
+
+
+ + + + + + + + + + + @foreach (var test in suite.Results) + { + + + + + + + } + +
Test CaseStatusResult DetailsMessage
+
@test.Name
+
@test.Description
+ 		+ + @(test.Result?.Status.ToString().ToUpper() ?? "SKIPPED") + + + @if (test.Result is NumericBenchmarkResult bsr) + { +
+
+
Mean: @bsr.Summary.Mean.ToString("F4")
+
Median: @bsr.Summary.Median.ToString("F4")
+
StdDev: @bsr.Summary.StdDev.ToString("F4")
+
+
+
Min: @bsr.Summary.Min.ToString("F4")
+
Max: @bsr.Summary.Max.ToString("F4")
+
+
+ } + else + { + var valProp = test.Result?.GetType().GetProperty("Value"); + object? val = null; + if (valProp != null) + { + val = valProp.GetValue(test.Result); + } + + if (val != null) + { + @val + } + } + + @if (test.Result is ErrorResult er && er.Exception != null) + { +
+ 
@er.Exception.ToString()
+
+ } + 		+ @test.Result?.Message +
+
+
+
+ } +
+ + diff --git a/Harp.Toolkit/Benchmark/Result.cs b/Harp.Toolkit/Benchmark/Result.cs new file mode 100644 index 0000000..312f97a --- /dev/null +++ b/Harp.Toolkit/Benchmark/Result.cs @@ -0,0 +1,161 @@ + +namespace Harp.Toolkit; + + +public enum Status +{ + Passed, + Failed, + Skipped, + Error +} + +public interface IResult +{ + string? Message { get; } + + Status Status { get; } +} + +public class ErrorResult(Exception exception) : IResult +{ + public string? Message { get; } = exception.Message; + public Status Status { get; } = Status.Error; + public Exception Exception { get; } = exception; +} + +public class Result : IResult +{ + public Result(T value, Status status, string message = "") + { + Status = status; + Value = value; + Message = message; + } + + public Result(T value, Func predicate, Func? messageFactory = null) + { + bool evaluation = predicate(value); + Status = evaluation ? Status.Passed : Status.Failed; + Value = value; + Message = messageFactory?.Invoke(value, evaluation) ?? string.Empty; + } + + + public string Message { get; } + public Status Status { get; } + public T Value { get; } + + public override string? ToString() + { + return $"Result(Status={Status}, Value={Value}, Message={Message})"; + } +} + + +public class AssertionResult : Result +{ + public AssertionResult(bool value, string message = "") + : base(value, value ? Status.Passed : Status.Failed, message) + { + } + + public AssertionResult(bool value, Func? messageFactory = null) + : base( + value, + v => v, + messageFactory is null ? null : ((value, evaluation) => messageFactory(value))) + { + + } +} + + +public class NumericBenchmarkResult : Result +{ + + public NumericBenchmarkResult(double[] values, Status status, string message = "") + : base(values, status, message) + { + Summary = new BenchmarkSummary(values); + } + + public NumericBenchmarkResult(BenchmarkSummary summary, Status status, string message = "") + : base(summary.Values, status, message) + { + Summary = summary; + } + + public NumericBenchmarkResult(double[] values, Func predicate, Func? messageFactory = null) + : base(values, predicate, messageFactory) + { + Summary = new BenchmarkSummary(values); + } + + public BenchmarkSummary Summary { get; } +} + + +public class BenchmarkSummary +{ + public readonly double[] Values; + + + public BenchmarkSummary(double[] values) + { + Values = values ?? Array.Empty(); + // TODO consider copying here since we are mutating + Array.Sort(Values); + } + + public double Mean => Values.Length == 0 ? double.NaN : Values.Average(); + + public double StdDev + { + get + { + if (Values.Length == 0) return double.NaN; + var mean = Mean; + var sumOfSquares = Values.Sum(v => (v - mean) * (v - mean)); + return Math.Sqrt(sumOfSquares / Values.Length); + } + } + + public double Median + { + get + { + if (Values.Length == 0) return double.NaN; + int mid = Values.Length / 2; + if (Values.Length % 2 == 0) + return (Values[mid - 1] + Values[mid]) / 2.0; + else + return Values[mid]; + } + } + + public double Max => Values.Length == 0 ? double.NaN : Values[Values.Length - 1]; + + public double Min => Values.Length == 0 ? double.NaN : Values[0]; + + public double Percentile99 => Percentile(0.99); + public double Percentile01 => Percentile(0.01); + + public double Percentile(double percentile) + { + if (Values.Length == 0) return double.NaN; + if (percentile < 0f || percentile > 1.0f) + { + throw new ArgumentOutOfRangeException(nameof(percentile), "Percentile must be between 0 and 1."); + } + + double rank = percentile * (Values.Length - 1); + int lower = (int)Math.Floor(rank); + int upper = (int)Math.Ceiling(rank); + if (lower == upper) return Values[lower]; + // Apparently this is how you solve rounding with percentiles + // https://en.wikipedia.org/wiki/Percentile + double weight = rank - lower; + return Values[lower] * (1 - weight) + Values[upper] * weight; + } +} diff --git a/Harp.Toolkit/Benchmark/Runner.cs b/Harp.Toolkit/Benchmark/Runner.cs new file mode 100644 index 0000000..ea062ba --- /dev/null +++ b/Harp.Toolkit/Benchmark/Runner.cs @@ -0,0 +1,53 @@ +using System.Runtime.CompilerServices; + +namespace Harp.Toolkit; + +public class Runner +{ + private readonly List suites = new(); + + public Runner() + { + } + + public int TestCount => suites.Sum(s => s.TestCount); + + public IEnumerable CollectSuites() + { + return suites.AsReadOnly(); + } + + public async IAsyncEnumerable<(Suite Suite, MethodResult Result)> RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (var suite in suites) + { + await foreach (var result in suite.RunAllAsync(portName, cancellationToken)) + { + yield return (suite, result); + } + } + } + + public void AddSuite(Suite suite) + { + if (suite == null) + { + throw new ArgumentNullException(nameof(suite)); + } + suites.Add(suite); + } + + public void ClearSuites() + { + suites.Clear(); + } + + public bool RemoveSuite(Suite suite) + { + if (suite == null) + { + throw new ArgumentNullException(nameof(suite)); + } + return suites.Remove(suite); + } +} diff --git a/Harp.Toolkit/Benchmark/Suite.cs b/Harp.Toolkit/Benchmark/Suite.cs new file mode 100644 index 0000000..5f27464 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Suite.cs @@ -0,0 +1,71 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using Bonsai.Harp; + +namespace Harp.Toolkit; + + +public abstract class Suite +{ + public abstract string Description { get; } + + public int TestCount => CollectTests().Count(); + + private IEnumerable<(MethodInfo Method, HarpTestAttribute Attribute)> CollectTests() + { + return GetType() + .GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) + .Select(m => (Method: m, Attribute: m.GetCustomAttribute()!)) + .Where(x => x.Attribute != null); + } + + public async IAsyncEnumerable RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (var (method, attr) in CollectTests()) + { + cancellationToken.ThrowIfCancellationRequested(); + + IResult testResult; + try + { + object? resultObj = method.Invoke(this, new object[] { portName }); + if (resultObj is Task task) + { + testResult = await task; + } + else if (resultObj is IResult syncResult) + { + testResult = syncResult; + } + else + { + throw new InvalidOperationException($"Test method '{method.Name}' must return IResult or Task."); + } + } + catch (Exception ex) + { + testResult = new ErrorResult(ex.InnerException ?? ex); + } + yield return new MethodResult + { + Result = testResult, + Name = method.Name, + Description = attr.Description ?? string.Empty + }; + } + } +} + +public class SuiteResult +{ + public required string Name { get; set; } + public string Description { get; set; } = string.Empty; + public List Results { get; set; } = new(); +} + +public class MethodResult +{ + public required string Name { get; set; } + public string Description { get; set; } = string.Empty; + public required IResult Result { get; set; } +} diff --git a/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs b/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs new file mode 100644 index 0000000..52031f7 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs @@ -0,0 +1,42 @@ + +using Bonsai.Harp; +namespace Harp.Toolkit; + +public class RoundTripTestSuite : Suite +{ + private double maxRoundTripDelayMs; + public RoundTripTestSuite(double maxRoundTripDelayMs = 4.0) + { + this.maxRoundTripDelayMs = maxRoundTripDelayMs; + } + + public override string Description => "A bunch of tests to benchmark round trip read/writes."; + + [HarpTest(Description = "Benchmarks the round trip time for a WhoAmI read command.")] + public async Task BenchmarkRoundTrip(string portName) + { + const int n = 1000; + double[] timestamps = new double[n]; + HarpMessage probe = WhoAmI.FromPayload(MessageType.Read, default); + using (var device = new AsyncDevice(portName)) + { + for (int i = 0; i < n; i++) + { + var reply = await device.CommandAsync(probe); + timestamps[i] = reply.GetTimestamp(); + } + } + var derivatives = timestamps + .Zip(timestamps.Skip(1), (previous, current) => (current - previous) * 1e3) + .ToArray(); + var benchmark = new BenchmarkSummary(derivatives); + if (benchmark.Max > maxRoundTripDelayMs) + { + return new NumericBenchmarkResult(benchmark, Status.Failed, $"Round trip WhoAmI read benchmark exceeded maximum allowed delay of {maxRoundTripDelayMs} ms."); + } + else + { + return new NumericBenchmarkResult(benchmark, Status.Passed, "Round trip WhoAmI read benchmark."); + } + } +} diff --git a/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs b/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs new file mode 100644 index 0000000..75f9266 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs @@ -0,0 +1,24 @@ + +using Bonsai.Harp; +namespace Harp.Toolkit; + +public class TimestampSecondsSuite : Suite +{ + public override string Description => "Timestamp Seconds Register Tests"; + + [HarpTest(Description = "Validates that the Timestamp Seconds register is writable.")] + public async Task IsWritable(string portName) + { + const uint setSeconds = 42; + using (var device = new AsyncDevice(portName)) + { + await device.WriteTimestampSecondsAsync(setSeconds); + await Task.Delay(1); + HarpMessage response = await device.CommandAsync(TimestampSeconds.FromPayload(MessageType.Read, default)); + double readSeconds = response.GetTimestamp(); + return new AssertionResult( + readSeconds - setSeconds < 1.0, + (success) => success ? $"`TimestampSeconds` register is writable and updates as expected." : $"`TimestampSeconds` register is not writable, Expected value: {setSeconds}, read value: {readSeconds}."); + } + } +} diff --git a/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs b/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs new file mode 100644 index 0000000..9954bc1 --- /dev/null +++ b/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs @@ -0,0 +1,21 @@ + +using Bonsai.Harp; +namespace Harp.Toolkit; + +public class WhoAmISuite : Suite +{ + public override string Description => "WhoAmI Register Tests"; + + [HarpTest(Description = "Validates that the WhoAmI register exists and contains a value.")] + public async Task CheckWhoAmI(string portName) + { + using (var device = new AsyncDevice(portName)) + { + int value = await device.ReadWhoAmIAsync(); + return new Result( + value, + (v) => v > 0 && v < 9999, + (v, success) => success ? $"WhoAmI register contains valid value: {v}." : $"WhoAmI register contains invalid value: {v}."); + } + } +} diff --git a/src/Harp.Toolkit/Program.cs b/src/Harp.Toolkit/Program.cs index f269381..de2dbfe 100644 --- a/src/Harp.Toolkit/Program.cs +++ b/src/Harp.Toolkit/Program.cs @@ -16,6 +16,7 @@ static async Task Main(string[] args) rootCommand.Subcommands.Add(new ListCommand()); rootCommand.Subcommands.Add(new UpdateFirmwareCommand()); rootCommand.Subcommands.Add(new GenerateCommand()); + rootCommand.Subcommands.Add(new BenchmarkCommand()); rootCommand.SetAction(async parseResult => { var portName = parseResult.GetRequiredValue(portNameOption); From b5a25ba1017c6dab68c63cb740afdb0b7b77d7cd Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Wed, 25 Mar 2026 09:02:03 -0700 Subject: [PATCH 02/41] Add dependencies --- src/Harp.Toolkit/Harp.Toolkit.csproj | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Harp.Toolkit/Harp.Toolkit.csproj b/src/Harp.Toolkit/Harp.Toolkit.csproj index 0fadfae..2c2931f 100644 --- a/src/Harp.Toolkit/Harp.Toolkit.csproj +++ b/src/Harp.Toolkit/Harp.Toolkit.csproj @@ -6,6 +6,7 @@ A tool for inspecting, updating and interfacing with Harp devices from the command-line. net8.0 enable + true @@ -14,6 +15,14 @@ + + - + + + PreserveNewest + + + + \ No newline at end of file From 4a4608ce70717becef5b8737d73461adeb2e39a3 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Wed, 25 Mar 2026 09:51:26 -0700 Subject: [PATCH 03/41] Refactor folder structure to latest version of the library --- {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/BenchmarkCommand.cs | 0 {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/HarpTestAttribute.cs | 0 .../Harp.Toolkit}/Benchmark/HtmlReportGenerator.cs | 0 {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Report.cs | 0 .../Harp.Toolkit}/Benchmark/ReportTemplate.cshtml | 0 {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Result.cs | 0 {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Runner.cs | 0 {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Suite.cs | 0 .../Harp.Toolkit}/Benchmark/Suites/RoundTripTestSuite.cs | 0 .../Harp.Toolkit}/Benchmark/Suites/TimestampSecondSuite.cs | 0 .../Harp.Toolkit}/Benchmark/Suites/WhoAmISuite.cs | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/BenchmarkCommand.cs (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/HarpTestAttribute.cs (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/HtmlReportGenerator.cs (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Report.cs (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/ReportTemplate.cshtml (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Result.cs (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Runner.cs (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Suite.cs (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Suites/RoundTripTestSuite.cs (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Suites/TimestampSecondSuite.cs (100%) rename {Harp.Toolkit => src/Harp.Toolkit}/Benchmark/Suites/WhoAmISuite.cs (100%) diff --git a/Harp.Toolkit/Benchmark/BenchmarkCommand.cs b/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs similarity index 100% rename from Harp.Toolkit/Benchmark/BenchmarkCommand.cs rename to src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs diff --git a/Harp.Toolkit/Benchmark/HarpTestAttribute.cs b/src/Harp.Toolkit/Benchmark/HarpTestAttribute.cs similarity index 100% rename from Harp.Toolkit/Benchmark/HarpTestAttribute.cs rename to src/Harp.Toolkit/Benchmark/HarpTestAttribute.cs diff --git a/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs b/src/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs similarity index 100% rename from Harp.Toolkit/Benchmark/HtmlReportGenerator.cs rename to src/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs diff --git a/Harp.Toolkit/Benchmark/Report.cs b/src/Harp.Toolkit/Benchmark/Report.cs similarity index 100% rename from Harp.Toolkit/Benchmark/Report.cs rename to src/Harp.Toolkit/Benchmark/Report.cs diff --git a/Harp.Toolkit/Benchmark/ReportTemplate.cshtml b/src/Harp.Toolkit/Benchmark/ReportTemplate.cshtml similarity index 100% rename from Harp.Toolkit/Benchmark/ReportTemplate.cshtml rename to src/Harp.Toolkit/Benchmark/ReportTemplate.cshtml diff --git a/Harp.Toolkit/Benchmark/Result.cs b/src/Harp.Toolkit/Benchmark/Result.cs similarity index 100% rename from Harp.Toolkit/Benchmark/Result.cs rename to src/Harp.Toolkit/Benchmark/Result.cs diff --git a/Harp.Toolkit/Benchmark/Runner.cs b/src/Harp.Toolkit/Benchmark/Runner.cs similarity index 100% rename from Harp.Toolkit/Benchmark/Runner.cs rename to src/Harp.Toolkit/Benchmark/Runner.cs diff --git a/Harp.Toolkit/Benchmark/Suite.cs b/src/Harp.Toolkit/Benchmark/Suite.cs similarity index 100% rename from Harp.Toolkit/Benchmark/Suite.cs rename to src/Harp.Toolkit/Benchmark/Suite.cs diff --git a/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs b/src/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs similarity index 100% rename from Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs rename to src/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs diff --git a/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs b/src/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs similarity index 100% rename from Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs rename to src/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs diff --git a/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs b/src/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs similarity index 100% rename from Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs rename to src/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs From 9a6a790422d6921e52ef5c68818b2f8dbe7421d6 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Sun, 19 Apr 2026 18:27:16 -0700 Subject: [PATCH 04/41] Modify organization to match register's name --- .../CoreRegisters/R_ASSEMBLY_VERSION.cs | 21 +++++++++++++++++++ .../R_TIMESTAMP_SECOND.cs} | 4 ++-- .../R_WHO_AM_I.cs} | 4 ++-- .../Benchmark/Suites/RoundTripTestSuite.cs | 6 +++--- 4 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs rename src/Harp.Toolkit/Benchmark/Suites/{TimestampSecondSuite.cs => CoreRegisters/R_TIMESTAMP_SECOND.cs} (92%) rename src/Harp.Toolkit/Benchmark/Suites/{WhoAmISuite.cs => CoreRegisters/R_WHO_AM_I.cs} (89%) diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs new file mode 100644 index 0000000..5e33221 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs @@ -0,0 +1,21 @@ + +using Bonsai.Harp; +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_ASSEMBLY_VERSION : Suite +{ + public override string Description => "WhoAmI Register Tests"; + + [HarpTest(Description = "Validates that the WhoAmI register exists and contains a value.")] + public async Task CheckWhoAmI(string portName) + { + using (var device = new AsyncDevice(portName)) + { + int value = await device.ReadWhoAmIAsync(); + return new Result( + value, + (v) => v > 0 && v < 9999, + (v, success) => success ? $"WhoAmI register contains valid value: {v}." : $"WhoAmI register contains invalid value: {v}."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs similarity index 92% rename from src/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs rename to src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs index 75f9266..1545913 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/TimestampSecondSuite.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs @@ -1,8 +1,8 @@  using Bonsai.Harp; -namespace Harp.Toolkit; +namespace Harp.Toolkit.Benchmark.Suites; -public class TimestampSecondsSuite : Suite +internal class R_TIMESTAMP_SECOND : Suite { public override string Description => "Timestamp Seconds Register Tests"; diff --git a/src/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs similarity index 89% rename from src/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs rename to src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs index 9954bc1..a64d047 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/WhoAmISuite.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs @@ -1,8 +1,8 @@  using Bonsai.Harp; -namespace Harp.Toolkit; +namespace Harp.Toolkit.Benchmark.Suites; -public class WhoAmISuite : Suite +internal class R_WHO_AM_I : Suite { public override string Description => "WhoAmI Register Tests"; diff --git a/src/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs b/src/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs index 52031f7..840b112 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs @@ -1,8 +1,8 @@  using Bonsai.Harp; -namespace Harp.Toolkit; +namespace Harp.Toolkit.Benchmark.Suites; -public class RoundTripTestSuite : Suite +internal class RoundTripTestSuite : Suite { private double maxRoundTripDelayMs; public RoundTripTestSuite(double maxRoundTripDelayMs = 4.0) @@ -17,7 +17,7 @@ public async Task BenchmarkRoundTrip(string portName) { const int n = 1000; double[] timestamps = new double[n]; - HarpMessage probe = WhoAmI.FromPayload(MessageType.Read, default); + HarpMessage probe = Bonsai.Harp.WhoAmI.FromPayload(MessageType.Read, default); using (var device = new AsyncDevice(portName)) { for (int i = 0; i < n; i++) From ed3127b74f776b752778b6b4f916bc8dea30bda0 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Sun, 19 Apr 2026 18:36:16 -0700 Subject: [PATCH 05/41] Add AssemblyRegister test --- .../Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs index 5e33221..197d06a 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs @@ -4,18 +4,20 @@ namespace Harp.Toolkit.Benchmark.Suites; internal class R_ASSEMBLY_VERSION : Suite { - public override string Description => "WhoAmI Register Tests"; + public override string Description => "AssemblyVersion Register Tests"; - [HarpTest(Description = "Validates that the WhoAmI register exists and contains a value.")] - public async Task CheckWhoAmI(string portName) + [HarpTest(Description = "Validates the deprecated register AssemblyVersion returns 0x00.")] + public async Task IsReturnZero(string portName) { using (var device = new AsyncDevice(portName)) { - int value = await device.ReadWhoAmIAsync(); - return new Result( - value, - (v) => v > 0 && v < 9999, - (v, success) => success ? $"WhoAmI register contains valid value: {v}." : $"WhoAmI register contains invalid value: {v}."); + int value = await device.ReadAssemblyVersionAsync(); + bool isZero = value == 0x00; + return new AssertionResult( + isZero, + isZero => isZero ? + $"AssemblyVersion register correctly returned 0x00." : + $"AssemblyVersion register returned a non-zero value (0x{value:X2})"); } } } From d0ae5806bb9e1a4c02449972010953039a69db6d Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Sun, 19 Apr 2026 19:07:43 -0700 Subject: [PATCH 06/41] Add R_UID tests --- .../CoreRegisters/R_ASSEMBLY_VERSION.cs | 4 +- .../Benchmark/Suites/CoreRegisters/R_UID.cs | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_UID.cs diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs index 197d06a..df07b42 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs @@ -7,7 +7,7 @@ internal class R_ASSEMBLY_VERSION : Suite public override string Description => "AssemblyVersion Register Tests"; [HarpTest(Description = "Validates the deprecated register AssemblyVersion returns 0x00.")] - public async Task IsReturnZero(string portName) + public async Task AssertReturnsZero(string portName) { using (var device = new AsyncDevice(portName)) { @@ -15,7 +15,7 @@ public async Task IsReturnZero(string portName) bool isZero = value == 0x00; return new AssertionResult( isZero, - isZero => isZero ? + x => x ? $"AssemblyVersion register correctly returned 0x00." : $"AssemblyVersion register returned a non-zero value (0x{value:X2})"); } diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_UID.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_UID.cs new file mode 100644 index 0000000..31a7fee --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_UID.cs @@ -0,0 +1,38 @@ + +using Bonsai.Harp; +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_UID : Suite +{ + private const byte address = 0x10; + private const byte expected_length = 16; + public override string Description => "UID Register Tests"; + + [HarpTest(Description = "Validates whether the UID register is 0 and thus likely not in use.")] + public async Task AssertLength(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var value = await device.ReadByteArrayAsync(address); + return new AssertionResult( + value.Length == expected_length, + x => x ? + $"Length is 16 as expected." : + $"Expected length of register to be 16, got {value.Length} instead"); + } + } + + [HarpTest(Description = "Checks if the register value is 0, indicating it is likely not used.")] + public async Task AssertReturnsZero(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var value = await device.ReadByteArrayAsync(address); + string msg = value.All(x => x == 0) ? "Value of all bytes is 0. Register likely not being used" : $"Register returned a non-zero value: {BitConverter.ToString(value)}"; + return new Result( + value, + Status.Passed, + msg); + } + } +} From bb5efdddbe63436e6d866b5d5328bf5867d95efd Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Sun, 19 Apr 2026 19:40:07 -0700 Subject: [PATCH 07/41] Add SerialNumber register test --- .../Suites/CoreRegisters/R_SERIAL_NUMBER.cs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_SERIAL_NUMBER.cs diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_SERIAL_NUMBER.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_SERIAL_NUMBER.cs new file mode 100644 index 0000000..65a475b --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_SERIAL_NUMBER.cs @@ -0,0 +1,28 @@ + +using Bonsai.Harp; +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_SERIAL_NUMBER : Suite +{ + public override string Description => "Serial Number Register Tests"; + + [HarpTest(Description = "Validates the contents of the register match the lower two bytes of R_UID")] + public async Task AssertConsitentWithUid(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var uidValue = await device.ReadByteArrayAsync(0x10); + if (uidValue.Length < 2) + throw new ArgumentException($"Expected UID register contents to be at least 2 bytes. Got {uidValue.Length}"); + var twoFirstBytes = BitConverter.ToInt16(uidValue, 0); + + var serialNumberValue = await device.ReadSerialNumberAsync(); + + return new AssertionResult( + twoFirstBytes == serialNumberValue, + x => x ? + $"SerialNumber register contents are consistent with UID register." : + $"SerialNumber register content (0x{serialNumberValue:X4}) does not match the first two bytes of UID register (0x{twoFirstBytes:X4})."); + } + } +} From 92a3624c257e5f4757bf24078922fa43fb2a2f09 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Sun, 19 Apr 2026 19:52:55 -0700 Subject: [PATCH 08/41] Add TimestampOffset register tests --- .../CoreRegisters/R_ASSEMBLY_VERSION.cs | 5 +-- .../CoreRegisters/R_TIMESTAMP_OFFSET.cs | 40 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs index df07b42..0089263 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs @@ -11,10 +11,9 @@ public async Task AssertReturnsZero(string portName) { using (var device = new AsyncDevice(portName)) { - int value = await device.ReadAssemblyVersionAsync(); - bool isZero = value == 0x00; + var value = await device.ReadAssemblyVersionAsync(); return new AssertionResult( - isZero, + value == 0x00, x => x ? $"AssemblyVersion register correctly returned 0x00." : $"AssemblyVersion register returned a non-zero value (0x{value:X2})"); diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs new file mode 100644 index 0000000..99a83c5 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs @@ -0,0 +1,40 @@ + +using Bonsai.Harp; +using System.Threading; +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_TIMESTAMP_OFFSET : Suite +{ + private const byte address = 0x0F; + public override string Description => "Timestamp Offset Register Tests"; + + [HarpTest(Description = "Validates the deprecated register TimestampOffset returns 0x00.")] + public async Task AssertReturnsZero(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var value = await device.ReadByteAsync(address); + return new AssertionResult( + value == 0x00, + x => x ? + $"TimestampOffset register correctly returned 0x00." : + $"TimestampOffset register returned a non-zero value (0x{value:X2})"); + } + } + + [HarpTest(Description = "Validates the deprecated register TimestampOffset is NOT writable.")] + public async Task IsNotWritable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var req = HarpMessage.FromByte(address, MessageType.Write, 0x00); + var value = await device.CommandAsync(req); + + return new AssertionResult( + value.Error, + x => x ? + $"Device correctly reported an error when trying to write to TimestampOffset register" : + $"Timestamp Offset register is deprecated and MUST NOT allow writes."); + } + } +} From 10119a697a4c898b8286a41bfa99b6dded64ab42 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 7 May 2026 09:49:12 -0700 Subject: [PATCH 09/41] Add additional tests for core registers --- .../Benchmark/BenchmarkCommand.cs | 66 ++++-- src/Harp.Toolkit/Benchmark/Runner.cs | 4 +- src/Harp.Toolkit/Benchmark/Suite.cs | 5 +- .../Suites/CoreRegisters/R_CLOCK_CONFIG.cs | 32 +++ .../Suites/CoreRegisters/R_CORE_VERSION_H.cs | 24 +++ .../Suites/CoreRegisters/R_CORE_VERSION_L.cs | 24 +++ .../Suites/CoreRegisters/R_DEVICE_NAME.cs | 36 ++++ .../Suites/CoreRegisters/R_FW_VERSION_H.cs | 24 +++ .../Suites/CoreRegisters/R_FW_VERSION_L.cs | 24 +++ .../Suites/CoreRegisters/R_HEARTBEAT.cs | 33 +++ .../Suites/CoreRegisters/R_HW_VERSION_H.cs | 24 +++ .../Suites/CoreRegisters/R_HW_VERSION_L.cs | 24 +++ .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 197 ++++++++++++++++++ .../Suites/CoreRegisters/R_RESET_DEV.cs | 18 ++ .../Benchmark/Suites/CoreRegisters/R_TAG.cs | 51 +++++ .../Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs | 56 +++++ .../CoreRegisters/R_TIMESTAMP_OFFSET.cs | 10 +- .../CoreRegisters/R_TIMESTAMP_SECOND.cs | 57 +++++ .../Suites/CoreRegisters/R_VERSION.cs | 51 +++++ .../Suites/CoreRegisters/R_WHO_AM_I.cs | 15 ++ .../Suites/CoreRegisters/_RegisterHelpers.cs | 50 +++++ 21 files changed, 793 insertions(+), 32 deletions(-) create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs diff --git a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs b/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs index f6754ef..9e67d89 100644 --- a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs +++ b/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs @@ -1,5 +1,6 @@ using System.CommandLine; using Spectre.Console; +using Harp.Toolkit.Benchmark.Suites; namespace Harp.Toolkit; public class BenchmarkCommand : Command @@ -42,29 +43,30 @@ static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verb RunDate = DateTime.Now }; - await AnsiConsole.Progress() - .StartAsync(async ctx => - { - var task = ctx.AddTask("[green]Running tests...[/]", true, runner.TestCount); + int currentTest = 0; + await foreach (var (suite, result) in runner.RunAllAsync(portName, cancellationToken, (suite, testName, testDesc) => + { + // Print "Running" status before test execution (without newline) + currentTest++; + Console.Write($"({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{testName} .... Running..."); + })) + { + // Clear the line by moving cursor to start and overwriting with spaces, then print result + Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r"); + AnsiConsole.MarkupLine($"[grey]({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}"); - await foreach (var (suite, result) in runner.RunAllAsync(portName, cancellationToken)) + var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name); + if (suiteResult == null) + { + suiteResult = new SuiteResult { - task.Increment(1); - AnsiConsole.MarkupLine($"[grey]{suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}"); - - var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name); - if (suiteResult == null) - { - suiteResult = new SuiteResult - { - Name = suite.GetType().Name, - Description = suite.Description - }; - report.Suites.Add(suiteResult); - } - suiteResult.Results.Add(result); - } - }); + Name = suite.GetType().Name, + Description = suite.Description + }; + report.Suites.Add(suiteResult); + } + suiteResult.Results.Add(result); + } if (verbose) { @@ -142,9 +144,27 @@ class CoreRunner : Runner { public CoreRunner() : base() { - AddSuite(new WhoAmISuite()); + AddSuite(new R_WHO_AM_I()); + AddSuite(new R_HW_VERSION_H()); + AddSuite(new R_HW_VERSION_L()); + AddSuite(new R_ASSEMBLY_VERSION()); + AddSuite(new R_CORE_VERSION_H()); + AddSuite(new R_CORE_VERSION_L()); + AddSuite(new R_FW_VERSION_H()); + AddSuite(new R_FW_VERSION_L()); + AddSuite(new R_TIMESTAMP_SECOND()); + AddSuite(new R_TIMESTAMP_MICRO()); + AddSuite(new R_OPERATION_CTRL()); + AddSuite(new R_RESET_DEV()); + AddSuite(new R_DEVICE_NAME()); + AddSuite(new R_SERIAL_NUMBER()); + AddSuite(new R_CLOCK_CONFIG()); + AddSuite(new R_TIMESTAMP_OFFSET()); + AddSuite(new R_UID()); + AddSuite(new R_TAG()); + AddSuite(new R_HEARTBEAT()); + AddSuite(new R_VERSION()); AddSuite(new RoundTripTestSuite()); - AddSuite(new TimestampSecondsSuite()); } } } diff --git a/src/Harp.Toolkit/Benchmark/Runner.cs b/src/Harp.Toolkit/Benchmark/Runner.cs index ea062ba..f243661 100644 --- a/src/Harp.Toolkit/Benchmark/Runner.cs +++ b/src/Harp.Toolkit/Benchmark/Runner.cs @@ -17,11 +17,11 @@ public IEnumerable CollectSuites() return suites.AsReadOnly(); } - public async IAsyncEnumerable<(Suite Suite, MethodResult Result)> RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable<(Suite Suite, MethodResult Result)> RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default, Action? onTestStart = null) { foreach (var suite in suites) { - await foreach (var result in suite.RunAllAsync(portName, cancellationToken)) + await foreach (var result in suite.RunAllAsync(portName, cancellationToken, (testName, testDesc) => onTestStart?.Invoke(suite, testName, testDesc))) { yield return (suite, result); } diff --git a/src/Harp.Toolkit/Benchmark/Suite.cs b/src/Harp.Toolkit/Benchmark/Suite.cs index 5f27464..3df8108 100644 --- a/src/Harp.Toolkit/Benchmark/Suite.cs +++ b/src/Harp.Toolkit/Benchmark/Suite.cs @@ -19,12 +19,15 @@ public abstract class Suite .Where(x => x.Attribute != null); } - public async IAsyncEnumerable RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default, Action? onTestStart = null) { foreach (var (method, attr) in CollectTests()) { cancellationToken.ThrowIfCancellationRequested(); + // Notify that test is starting + onTestStart?.Invoke(method.Name, attr.Description ?? string.Empty); + IResult testResult; try { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs new file mode 100644 index 0000000..637c346 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs @@ -0,0 +1,32 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_CLOCK_CONFIG : Suite +{ + private const byte address = 0x0E; + public override string Description => "Clock Configuration Register Tests"; + + [HarpTest(Description = "Validates that ClockConfig register is readable.")] + public async Task IsReadable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + return await RegisterHelpers.AssertReadableByteAsync(device, address, "ClockConfig"); + } + } + + [HarpTest(Description = "Reports clock synchronization capability: REP_ABLE (bit 3) and GEN_ABLE (bit 4).")] + public async Task ReportSyncCapability(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var value = await device.ReadByteAsync(address); + bool repAble = (value & (1 << 3)) != 0; + bool genAble = (value & (1 << 4)) != 0; + return new AssertionResult( + true, + $"ClockConfig sync capability: REP_ABLE={repAble}, GEN_ABLE={genAble}."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs new file mode 100644 index 0000000..bffcd9f --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs @@ -0,0 +1,24 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_CORE_VERSION_H : Suite +{ + private const byte address = 0x04; + public override string Description => "Core Version High Register Tests"; + + [HarpTest(Description = "Validates that CoreVersionHigh matches byte 0 of R_VERSION.")] + public async Task AssertConsistentWithVersion(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var versionArray = await device.ReadByteArrayAsync(0x13); + var registerValue = await device.ReadByteAsync(address); + return new AssertionResult( + registerValue == versionArray[0], + x => x + ? $"CoreVersionHigh (0x{registerValue:X2}) matches R_VERSION byte 0." + : $"CoreVersionHigh (0x{registerValue:X2}) does not match R_VERSION byte 0 (0x{versionArray[0]:X2})."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs new file mode 100644 index 0000000..a22f4ee --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs @@ -0,0 +1,24 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_CORE_VERSION_L : Suite +{ + private const byte address = 0x05; + public override string Description => "Core Version Low Register Tests"; + + [HarpTest(Description = "Validates that CoreVersionLow matches byte 1 of R_VERSION.")] + public async Task AssertConsistentWithVersion(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var versionArray = await device.ReadByteArrayAsync(0x13); + var registerValue = await device.ReadByteAsync(address); + return new AssertionResult( + registerValue == versionArray[1], + x => x + ? $"CoreVersionLow (0x{registerValue:X2}) matches R_VERSION byte 1." + : $"CoreVersionLow (0x{registerValue:X2}) does not match R_VERSION byte 1 (0x{versionArray[1]:X2})."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs new file mode 100644 index 0000000..3bdf6e8 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs @@ -0,0 +1,36 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_DEVICE_NAME : Suite +{ + private const byte address = 0x0C; + private const int expectedLength = 25; + public override string Description => "Device Name Register Tests"; + + [HarpTest(Description = "Validates that DeviceName register is readable.")] + public async Task IsReadable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + try + { + await device.ReadByteArrayAsync(address); + return new AssertionResult(true, "DeviceName is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + } + } + + [HarpTest(Description = "Validates that DeviceName register has exactly 25 bytes.")] + public async Task AssertLength(string portName) + { + using (var device = new AsyncDevice(portName)) + { + return await RegisterHelpers.AssertReadableArrayAsync(device, address, expectedLength, "DeviceName"); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs new file mode 100644 index 0000000..9637d3b --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs @@ -0,0 +1,24 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_FW_VERSION_H : Suite +{ + private const byte address = 0x06; + public override string Description => "Firmware Version High Register Tests"; + + [HarpTest(Description = "Validates that FwVersionHigh matches byte 3 of R_VERSION.")] + public async Task AssertConsistentWithVersion(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var versionArray = await device.ReadByteArrayAsync(0x13); + var registerValue = await device.ReadByteAsync(address); + return new AssertionResult( + registerValue == versionArray[3], + x => x + ? $"FwVersionHigh (0x{registerValue:X2}) matches R_VERSION byte 3." + : $"FwVersionHigh (0x{registerValue:X2}) does not match R_VERSION byte 3 (0x{versionArray[3]:X2})."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs new file mode 100644 index 0000000..d3862d0 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs @@ -0,0 +1,24 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_FW_VERSION_L : Suite +{ + private const byte address = 0x07; + public override string Description => "Firmware Version Low Register Tests"; + + [HarpTest(Description = "Validates that FwVersionLow matches byte 4 of R_VERSION.")] + public async Task AssertConsistentWithVersion(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var versionArray = await device.ReadByteArrayAsync(0x13); + var registerValue = await device.ReadByteAsync(address); + return new AssertionResult( + registerValue == versionArray[4], + x => x + ? $"FwVersionLow (0x{registerValue:X2}) matches R_VERSION byte 4." + : $"FwVersionLow (0x{registerValue:X2}) does not match R_VERSION byte 4 (0x{versionArray[4]:X2})."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs new file mode 100644 index 0000000..25a7322 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs @@ -0,0 +1,33 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_HEARTBEAT : Suite +{ + private const byte address = 0x12; + public override string Description => "Heartbeat Register Tests"; + + [HarpTest(Description = "Validates that Heartbeat register is readable.")] + public async Task IsReadable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + return await RegisterHelpers.AssertReadableByteAsync(device, address, "Heartbeat"); + } + } + + [HarpTest(Description = "Validates that Heartbeat register is NOT writable.")] + public async Task IsNotWritable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var req = HarpMessage.FromByte(address, MessageType.Write, 0x00); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x + ? "Heartbeat register correctly rejected write." + : "Heartbeat register should NOT be writable."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs new file mode 100644 index 0000000..e1d70e8 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs @@ -0,0 +1,24 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_HW_VERSION_H : Suite +{ + private const byte address = 0x01; + public override string Description => "Hardware Version High Register Tests"; + + [HarpTest(Description = "Validates that HwVersionHigh matches byte 6 of R_VERSION.")] + public async Task AssertConsistentWithVersion(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var versionArray = await device.ReadByteArrayAsync(0x13); + var registerValue = await device.ReadByteAsync(address); + return new AssertionResult( + registerValue == versionArray[6], + x => x + ? $"HwVersionHigh (0x{registerValue:X2}) matches R_VERSION byte 6." + : $"HwVersionHigh (0x{registerValue:X2}) does not match R_VERSION byte 6 (0x{versionArray[6]:X2})."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs new file mode 100644 index 0000000..a4e1b16 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs @@ -0,0 +1,24 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_HW_VERSION_L : Suite +{ + private const byte address = 0x02; + public override string Description => "Hardware Version Low Register Tests"; + + [HarpTest(Description = "Validates that HwVersionLow matches byte 7 of R_VERSION.")] + public async Task AssertConsistentWithVersion(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var versionArray = await device.ReadByteArrayAsync(0x13); + var registerValue = await device.ReadByteAsync(address); + return new AssertionResult( + registerValue == versionArray[7], + x => x + ? $"HwVersionLow (0x{registerValue:X2}) matches R_VERSION byte 7." + : $"HwVersionLow (0x{registerValue:X2}) does not match R_VERSION byte 7 (0x{versionArray[7]:X2})."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs new file mode 100644 index 0000000..b98eda1 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -0,0 +1,197 @@ +using Bonsai.Harp; +using System.Reactive.Linq; +using System.Collections.Concurrent; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_OPERATION_CTRL : Suite +{ + private const byte address = 0x0A; + public override string Description => "Operation Control Register Tests"; + + [HarpTest(Description = "Validates that OP_MODE bits can be round-tripped between Standby (0) and Active (1).")] + public async Task OpModeRoundTrip(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var original = await device.ReadByteAsync(address); + byte currentMode = (byte)(original & 0x03); + byte newMode = currentMode == 0x01 ? (byte)0x00 : (byte)0x01; + byte newValue = (byte)((original & ~0x03) | newMode); + + try + { + await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, newValue)); + var readBack = await device.ReadByteAsync(address); + byte readMode = (byte)(readBack & 0x03); + + return new AssertionResult( + readMode == newMode, + x => x + ? $"OpModeRoundTrip: OP_MODE correctly round-tripped to {newMode}." + : $"OpModeRoundTrip: wrote OP_MODE={newMode}, read back OP_MODE={readMode}."); + } + finally + { + // Always restore original state + try + { + await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, original)); + } + catch + { + // Ignore errors during restoration + } + } + } + } + + [HarpTest(Description = "Validates that ALIVE_EN (deprecated, bit 7) can be toggled, or reports as unsupported.")] + public async Task AliveEnWritable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + return await TestOptionalBitAsync(device, "AliveEn", 0x80); + } + } + + [HarpTest(Description = "Validates that OPLED_EN (optional, bit 6) can be toggled, or reports as unsupported.")] + public async Task OpLedEnWritable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + return await TestOptionalBitAsync(device, "OpLedEn", 0x40); + } + } + + [HarpTest(Description = "Validates that VISUAL_EN (optional, bit 5) can be toggled, or reports as unsupported.")] + public async Task VisualEnWritable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + return await TestOptionalBitAsync(device, "VisualEn", 0x20); + } + } + + [HarpTest(Description = "Validates that enabling HEARTBEAT_EN causes the device to emit R_HEARTBEAT events.")] + public async Task HeartbeatEnEmitsEvents(string portName) + { + byte originalOpCtrl = 0; + IDisposable? subscription = null; + + try + { + // Read original state before modifying + using (var device = new AsyncDevice(portName)) + { + originalOpCtrl = await device.ReadByteAsync(address); + } + + var harpDevice = new Bonsai.Harp.Device { PortName = portName, Heartbeat = EnableFlag.Enabled }; + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + cts.Token.Register(() => tcs.TrySetResult(false)); + + subscription = harpDevice.Generate() + .Where(m => m.Address == 0x12 && m.MessageType == MessageType.Event) + .Take(1) + .Subscribe( + _ => tcs.TrySetResult(true), + ex => tcs.TrySetException(ex)); + + bool received = await tcs.Task; + + return new AssertionResult( + received, + x => x + ? "HeartbeatEnEmitsEvents: heartbeat event received within 2s." + : "HeartbeatEnEmitsEvents: no heartbeat event received within 2s."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + finally + { + subscription?.Dispose(); + await Task.Delay(200); // Fudge delay to ensure port is released + + // Always restore original Operation Control state + using (var device = new AsyncDevice(portName)) + { + await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, originalOpCtrl)); + } + + } + } + + [HarpTest(Description = "Validates that the DUMP bit triggers a burst of all core register reads after an OpCtrl write.")] + public async Task DumpEmitsRegisterBurst(string portName) + { + byte originalOpCtrl = 0; + var messages = new ConcurrentQueue(); + IDisposable? subscription = null; + + try + { + // Read original state before modifying + using (var device = new AsyncDevice(portName)) + { + originalOpCtrl = await device.ReadByteAsync(address); + } + + var harpDevice = new Bonsai.Harp.Device { PortName = portName, DumpRegisters = true }; + subscription = harpDevice.Generate() + .Subscribe(m => messages.Enqueue(m)); + + await Task.Delay(1000); + + var snapshot = messages.ToList(); + + int opCtrlWriteIdx = -1; + for (int i = 0; i < snapshot.Count; i++) + { + if (snapshot[i].Address == 0x0A && snapshot[i].MessageType == MessageType.Write) + { + opCtrlWriteIdx = i; + break; + } + } + + if (opCtrlWriteIdx < 0) + return new AssertionResult(false, "DumpEmitsRegisterBurst: no Write reply at OpCtrl (0x0A) found."); + + var coreReads = snapshot + .Select((m, i) => (msg: m, idx: i)) + .Where(x => x.msg.Address <= 0x13 && x.msg.MessageType == MessageType.Read) + .ToList(); + + bool writeBeforeAllReads = coreReads.All(x => opCtrlWriteIdx < x.idx); + if (!writeBeforeAllReads) + return new AssertionResult(false, "DumpEmitsRegisterBurst: OpCtrl Write reply did not precede all core Read replies."); + + var presentAddresses = coreReads.Select(x => (int)x.msg.Address).Distinct().ToHashSet(); + var missing = Enumerable.Range(0, 0x14).Where(a => !presentAddresses.Contains(a)).ToList(); + if (missing.Count > 0) + return new AssertionResult(false, + $"DumpEmitsRegisterBurst: missing Read replies for {missing.Count} core address(es): {string.Join(", ", missing.Select(a => $"0x{a:X2}"))}."); + + return new AssertionResult(true, "DumpEmitsRegisterBurst: all 20 core register reads received after OpCtrl write."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + finally + { + subscription?.Dispose(); + await Task.Delay(200); + + // Ensure we restore original state even though DUMP is transient + using (var device = new AsyncDevice(portName)) + { + await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, originalOpCtrl)); + } + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs new file mode 100644 index 0000000..3aabc6f --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs @@ -0,0 +1,18 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_RESET_DEV : Suite +{ + private const byte address = 0x0B; + public override string Description => "Reset Device Register Tests"; + + [HarpTest(Description = "Validates that ResetDev register is readable.")] + public async Task IsReadable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + return await RegisterHelpers.AssertReadableByteAsync(device, address, "ResetDev"); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs new file mode 100644 index 0000000..3a4f291 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs @@ -0,0 +1,51 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_TAG : Suite +{ + private const byte address = 0x11; + private const int expectedLength = 8; + public override string Description => "Tag Register Tests"; + + [HarpTest(Description = "Validates that Tag register is readable.")] + public async Task IsReadable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + try + { + await device.ReadByteArrayAsync(address); + return new AssertionResult(true, "Tag is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + } + } + + [HarpTest(Description = "Validates that Tag register has exactly 8 bytes.")] + public async Task AssertLength(string portName) + { + using (var device = new AsyncDevice(portName)) + { + return await RegisterHelpers.AssertReadableArrayAsync(device, address, expectedLength, "Tag"); + } + } + + [HarpTest(Description = "Validates that Tag register is NOT writable.")] + public async Task IsNotWritable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var req = HarpMessage.FromByte(address, MessageType.Write, 0x00); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x + ? "Tag register correctly rejected write." + : "Tag register should NOT be writable."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs new file mode 100644 index 0000000..f863303 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs @@ -0,0 +1,56 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_TIMESTAMP_MICRO : Suite +{ + private const byte address = 0x09; + public override string Description => "Timestamp Microseconds Register Tests"; + + [HarpTest(Description = "Validates that TimestampMicro register is readable.")] + public async Task IsReadable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + try + { + await device.ReadByteArrayAsync(address); + return new AssertionResult(true, "TimestampMicro is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + } + } + + [HarpTest(Description = "Validates that TimestampMicro register is NOT writable.")] + public async Task IsNotWritable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var req = HarpMessage.FromUInt16(address, MessageType.Write, 0); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x + ? "TimestampMicro register correctly rejected write." + : "TimestampMicro register should NOT be writable."); + } + } + + [HarpTest(Description = "Validates that TimestampMicro value is within bounds (0 to 31249).")] + public async Task ValueWithinBounds(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var rawBytes = await device.ReadByteArrayAsync(address); + var microValue = BitConverter.ToUInt16(rawBytes, 0); + return new AssertionResult( + microValue < 31250, + x => x + ? $"TimestampMicro value ({microValue}) is within expected bounds (< 31250)." + : $"TimestampMicro value ({microValue}) exceeds expected maximum (31249)."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs index 99a83c5..e67623c 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs @@ -1,6 +1,5 @@  using Bonsai.Harp; -using System.Threading; namespace Harp.Toolkit.Benchmark.Suites; internal class R_TIMESTAMP_OFFSET : Suite @@ -28,13 +27,12 @@ public async Task IsNotWritable(string portName) using (var device = new AsyncDevice(portName)) { var req = HarpMessage.FromByte(address, MessageType.Write, 0x00); - var value = await device.CommandAsync(req); - + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( - value.Error, + rejected, x => x ? - $"Device correctly reported an error when trying to write to TimestampOffset register" : - $"Timestamp Offset register is deprecated and MUST NOT allow writes."); + "Device correctly reported an error when trying to write to TimestampOffset register." : + "Timestamp Offset register is deprecated and MUST NOT allow writes."); } } } diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs index 1545913..489be6d 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs @@ -1,5 +1,6 @@  using Bonsai.Harp; +using System.Diagnostics; namespace Harp.Toolkit.Benchmark.Suites; internal class R_TIMESTAMP_SECOND : Suite @@ -21,4 +22,60 @@ public async Task IsWritable(string portName) (success) => success ? $"`TimestampSeconds` register is writable and updates as expected." : $"`TimestampSeconds` register is not writable, Expected value: {setSeconds}, read value: {readSeconds}."); } } + + [HarpTest(Description = "Validates that TimestampSeconds register is readable.")] + public async Task IsReadable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + try + { + await device.ReadTimestampSecondsAsync(); + return new AssertionResult(true, "TimestampSeconds is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + } + } + + [HarpTest(Description = "Validates that TimestampSeconds register is monotonically non-decreasing.")] + public async Task IsMonotonic(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var first = await device.ReadTimestampSecondsAsync(); + await Task.Delay(100); + var second = await device.ReadTimestampSecondsAsync(); + return new AssertionResult( + second >= first, + x => x + ? $"TimestampSeconds is monotonic: {first} -> {second}." + : $"TimestampSeconds decreased from {first} to {second}."); + } + } + + [HarpTest(Description = "Validates that writing a past timestamp value takes effect and can be read back.")] + public async Task WritePastValueRoundTrip(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var sw = Stopwatch.StartNew(); + var current = await device.ReadTimestampSecondsAsync(); + var tPast = current >= 10 ? current - 10 : 0u; + + await device.WriteTimestampSecondsAsync(tPast); + await Task.Delay(50); + + var readBack = await device.ReadTimestampSecondsAsync(); + bool withinBounds = Math.Abs((long)readBack - (long)tPast) <= 1; + + return new AssertionResult( + withinBounds, + x => x + ? $"WritePastValueRoundTrip: wrote {tPast}, read back {readBack} (within 1s tolerance)." + : $"WritePastValueRoundTrip: wrote {tPast}, read back {readBack} (difference {Math.Abs((long)readBack - (long)tPast)}s, expected <= 1)."); + } + } } diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs new file mode 100644 index 0000000..68b01cd --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs @@ -0,0 +1,51 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class R_VERSION : Suite +{ + private const byte address = 0x13; + private const int expectedLength = 32; + public override string Description => "Version Register Tests"; + + [HarpTest(Description = "Validates that Version register is readable.")] + public async Task IsReadable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + try + { + await device.ReadByteArrayAsync(address); + return new AssertionResult(true, "Version is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + } + } + + [HarpTest(Description = "Validates that Version register has exactly 32 bytes.")] + public async Task AssertLength(string portName) + { + using (var device = new AsyncDevice(portName)) + { + return await RegisterHelpers.AssertReadableArrayAsync(device, address, expectedLength, "Version"); + } + } + + [HarpTest(Description = "Validates that Version register is NOT writable.")] + public async Task IsNotWritable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var req = HarpMessage.FromByte(address, MessageType.Write, 0x00); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x + ? "Version register correctly rejected write." + : "Version register should NOT be writable."); + } + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs index a64d047..368fae3 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs @@ -18,4 +18,19 @@ public async Task CheckWhoAmI(string portName) (v, success) => success ? $"WhoAmI register contains valid value: {v}." : $"WhoAmI register contains invalid value: {v}."); } } + + [HarpTest(Description = "Validates that the WhoAmI register is NOT writable.")] + public async Task IsNotWritable(string portName) + { + using (var device = new AsyncDevice(portName)) + { + var req = HarpMessage.FromUInt16(0x00, MessageType.Write, 0); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x ? + "WhoAmI register correctly rejected write." : + "WhoAmI register should NOT be writable."); + } + } } diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs new file mode 100644 index 0000000..a465759 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs @@ -0,0 +1,50 @@ + +using Bonsai.Harp; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal static class RegisterHelpers +{ + public static async Task IsWriteRejectedAsync(AsyncDevice device, HarpMessage write) + { + try + { + await device.CommandAsync(write); + return false; + } + catch (HarpException) + { + return true; + } + } + + public static async Task AssertReadableArrayAsync(AsyncDevice device, int address, int expectedLength, string registerName) + { + try + { + var value = await device.ReadByteArrayAsync(address); + return new AssertionResult( + value.Length == expectedLength, + x => x + ? $"{registerName} is readable and has expected length ({expectedLength})." + : $"{registerName} returned {value.Length} bytes, expected {expectedLength}."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + } + + public static async Task AssertReadableByteAsync(AsyncDevice device, int address, string registerName) + { + try + { + await device.ReadByteAsync(address); + return new AssertionResult(true, $"{registerName} is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + } +} From f16d51a3f14201dfffe5ec66e5a95fe5f0c0f7ae Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 7 May 2026 13:54:38 -0700 Subject: [PATCH 10/41] Fix register type --- .../Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs index f863303..3fdc73f 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; @@ -14,7 +14,7 @@ public async Task IsReadable(string portName) { try { - await device.ReadByteArrayAsync(address); + await device.ReadUInt16Async(address); return new AssertionResult(true, "TimestampMicro is readable."); } catch (Exception ex) @@ -44,8 +44,7 @@ public async Task ValueWithinBounds(string portName) { using (var device = new AsyncDevice(portName)) { - var rawBytes = await device.ReadByteArrayAsync(address); - var microValue = BitConverter.ToUInt16(rawBytes, 0); + var microValue = await device.ReadUInt16Async(address); return new AssertionResult( microValue < 31250, x => x From f2f6f6dfaacb9bbb8020d1c5c4cb20202ba4d3b8 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 7 May 2026 15:28:10 -0700 Subject: [PATCH 11/41] Add wrapper for stream-able writes --- .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 70 +++++++++++++------ .../Suites/CoreRegisters/_RegisterHelpers.cs | 40 ++++++++++- 2 files changed, 89 insertions(+), 21 deletions(-) diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs index b98eda1..223ba15 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -77,29 +77,25 @@ public async Task VisualEnWritable(string portName) public async Task HeartbeatEnEmitsEvents(string portName) { byte originalOpCtrl = 0; - IDisposable? subscription = null; try { - // Read original state before modifying using (var device = new AsyncDevice(portName)) { originalOpCtrl = await device.ReadByteAsync(address); } + await Task.Delay(500); // The previous one needs some time to disconnect + + var harpDevice = new Bonsai.Harp.Device { PortName = portName }; + var responses = await RegisterHelpers.WriteToTransportAsync( + portName, + new[] { HarpMessage.FromByte(address, MessageType.Write, 0xE5) }, + TimeSpan.FromSeconds(0.5)); + var messages = await harpDevice.Generate() + .TakeUntil(Observable.Timer(TimeSpan.FromSeconds(2))) + .ToList(); - var harpDevice = new Bonsai.Harp.Device { PortName = portName, Heartbeat = EnableFlag.Enabled }; - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); - cts.Token.Register(() => tcs.TrySetResult(false)); - - subscription = harpDevice.Generate() - .Where(m => m.Address == 0x12 && m.MessageType == MessageType.Event) - .Take(1) - .Subscribe( - _ => tcs.TrySetResult(true), - ex => tcs.TrySetException(ex)); - - bool received = await tcs.Task; + bool received = messages.Any(m => m.Address == 0x18 && m.MessageType == MessageType.Event); return new AssertionResult( received, @@ -113,15 +109,11 @@ public async Task HeartbeatEnEmitsEvents(string portName) } finally { - subscription?.Dispose(); - await Task.Delay(200); // Fudge delay to ensure port is released - - // Always restore original Operation Control state + await Task.Delay(200); // Wait for port to be released before reopening using (var device = new AsyncDevice(portName)) { await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, originalOpCtrl)); } - } } @@ -194,4 +186,42 @@ public async Task DumpEmitsRegisterBurst(string portName) } } } + + private static async Task TestOptionalBitAsync(AsyncDevice device, string bitName, byte bitMask) + { + var original = await device.ReadByteAsync(address); + byte toggled = (byte)(original ^ bitMask); + + try + { + try + { + await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, toggled)); + } + catch (HarpException) + { + return new Result(false, Status.Skipped, + $"{bitName} is optional/deprecated and not supported by this device."); + } + + var readBack = await device.ReadByteAsync(address); + bool bitChanged = (readBack & bitMask) == (toggled & bitMask); + + return new AssertionResult( + bitChanged, + x => x + ? $"{bitName}: bit correctly toggled." + : $"{bitName}: bit did not change after write (expected {(toggled & bitMask) != 0}, got {(readBack & bitMask) != 0})."); + } + finally + { + try + { + await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, original)); + } + catch + { + } + } + } } diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs index a465759..0179719 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs @@ -1,10 +1,48 @@ - + using Bonsai.Harp; +using System.Reactive.Linq; +using System.Reactive.Subjects; namespace Harp.Toolkit.Benchmark.Suites; internal static class RegisterHelpers { + /// + /// Opens a Device connection, writes messages via the synchronous transport, + /// collects all received messages for the specified duration, then cleans up. + /// + public static async Task> WriteToTransportAsync( + string portName, + IEnumerable messagesToWrite, + TimeSpan listenDuration, + Action? configureDevice = null) + { + var harpDevice = new Bonsai.Harp.Device { PortName = portName }; + configureDevice?.Invoke(harpDevice); + + var source = new Subject(); + var collected = new List(); + var tcs = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + + using var subscription = harpDevice.Generate(source) + .Subscribe( + onNext: m => collected.Add(m), + onError: ex => tcs.TrySetException(ex)); + + // Small delay to let the transport connect + await Task.Delay(200); + + foreach (var msg in messagesToWrite) + { + source.OnNext(msg); + } + + await Task.Delay(listenDuration); + + source.OnCompleted(); + tcs.TrySetResult(collected); + return await tcs.Task; + } public static async Task IsWriteRejectedAsync(AsyncDevice device, HarpMessage write) { try From a8d53e63b5934345b6f389e212f8e2db5c6912ee Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 7 May 2026 15:28:58 -0700 Subject: [PATCH 12/41] Escape potential special characters --- src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs b/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs index 9e67d89..c2749e7 100644 --- a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs +++ b/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs @@ -107,10 +107,10 @@ static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verb } table.AddRow( - new Markup($"[bold]{test.Name}[/]\n[dim]{test.Description}[/]"), + new Markup($"[bold]{Markup.Escape(test.Name)}[/]\n[dim]{Markup.Escape(test.Description)}[/]"), new Markup(GetResultMarkup(test.Result)), - new Markup(details), - new Markup(message) + new Markup(Markup.Escape(details)), + new Markup(Markup.Escape(message)) ); } AnsiConsole.Write(table); From f26932a523a79bf155925276ff25fca2933507d7 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 7 May 2026 15:56:27 -0700 Subject: [PATCH 13/41] Make method generic and pass delegate --- .../Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs | 11 +++++++++-- .../Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs | 6 +++--- .../Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs | 4 ++-- .../Suites/CoreRegisters/_RegisterHelpers.cs | 4 ++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs index 637c346..50b044e 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs @@ -1,3 +1,4 @@ +using System.Text; using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; @@ -12,7 +13,7 @@ public async Task IsReadable(string portName) { using (var device = new AsyncDevice(portName)) { - return await RegisterHelpers.AssertReadableByteAsync(device, address, "ClockConfig"); + return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), address, "ClockConfig"); } } @@ -24,9 +25,15 @@ public async Task ReportSyncCapability(string portName) var value = await device.ReadByteAsync(address); bool repAble = (value & (1 << 3)) != 0; bool genAble = (value & (1 << 4)) != 0; + StringBuilder sb = new StringBuilder("ClockConfig sync capability:"); + sb.Append("\n"); + sb.Append(repAble ? "Device can repeat clock signal" : "Device cannot repeat clock signal"); + sb.Append("\n"); + sb.Append(genAble ? "Device can generate clock signal" : "Device cannot generate clock signal"); + sb.Append("\n"); return new AssertionResult( true, - $"ClockConfig sync capability: REP_ABLE={repAble}, GEN_ABLE={genAble}."); + sb.ToString()); } } } diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs index 25a7322..ffb3e4c 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs @@ -1,10 +1,10 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; internal class R_HEARTBEAT : Suite { - private const byte address = 0x12; + private const byte address = 18; public override string Description => "Heartbeat Register Tests"; [HarpTest(Description = "Validates that Heartbeat register is readable.")] @@ -12,7 +12,7 @@ public async Task IsReadable(string portName) { using (var device = new AsyncDevice(portName)) { - return await RegisterHelpers.AssertReadableByteAsync(device, address, "Heartbeat"); + return await RegisterHelpers.AssertReadableAsync(a => device.ReadUInt16Async(a), address, "Heartbeat"); } } diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs index 3aabc6f..4e80a83 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; @@ -12,7 +12,7 @@ public async Task IsReadable(string portName) { using (var device = new AsyncDevice(portName)) { - return await RegisterHelpers.AssertReadableByteAsync(device, address, "ResetDev"); + return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), address, "ResetDev"); } } } diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs index 0179719..dfb82f8 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs @@ -73,11 +73,11 @@ public static async Task AssertReadableArrayAsync(AsyncDevice device, i } } - public static async Task AssertReadableByteAsync(AsyncDevice device, int address, string registerName) + public static async Task AssertReadableAsync(Func> readFunc, int address, string registerName) { try { - await device.ReadByteAsync(address); + await readFunc(address); return new AssertionResult(true, $"{registerName} is readable."); } catch (Exception ex) From 63d2c689b0f047e25149fa0345fb5a79ae286d48 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 7 May 2026 16:21:47 -0700 Subject: [PATCH 14/41] Format --- .../Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs | 2 +- .../Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs | 2 +- .../Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs | 2 +- .../Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs | 2 +- .../Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs | 2 +- .../Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs | 2 +- .../Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs | 2 +- src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs | 2 +- src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs index bffcd9f..6bc733a 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs index a22f4ee..7be1036 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs index 3bdf6e8..f0d516c 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs index 9637d3b..0808222 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs index d3862d0..637e7a1 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs index e1d70e8..41bff61 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs index a4e1b16..fdd328c 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs index 3a4f291..7406bfe 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs index 68b01cd..e914bf4 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Benchmark.Suites; From 3ef0b1edea34f4cbd53e6af41bc9cc543755f68a Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 7 May 2026 16:22:43 -0700 Subject: [PATCH 15/41] Favor async transport for writing operation control state --- .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 49 ++++++------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs index 223ba15..9f54c70 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -118,11 +118,9 @@ public async Task HeartbeatEnEmitsEvents(string portName) } [HarpTest(Description = "Validates that the DUMP bit triggers a burst of all core register reads after an OpCtrl write.")] - public async Task DumpEmitsRegisterBurst(string portName) + public async Task RegisterDump(string portName) { byte originalOpCtrl = 0; - var messages = new ConcurrentQueue(); - IDisposable? subscription = null; try { @@ -132,43 +130,28 @@ public async Task DumpEmitsRegisterBurst(string portName) originalOpCtrl = await device.ReadByteAsync(address); } - var harpDevice = new Bonsai.Harp.Device { PortName = portName, DumpRegisters = true }; - subscription = harpDevice.Generate() - .Subscribe(m => messages.Enqueue(m)); - - await Task.Delay(1000); - - var snapshot = messages.ToList(); + var harpDevice = new Bonsai.Harp.Device { PortName = portName }; + var messages = await RegisterHelpers.WriteToTransportAsync( + portName, + new[] { HarpMessage.FromByte(address, MessageType.Write, (byte)(originalOpCtrl | 0x08)) }, + TimeSpan.FromSeconds(1)); - int opCtrlWriteIdx = -1; - for (int i = 0; i < snapshot.Count; i++) + var opRegWriteResponse = messages.FirstOrDefault(m => m.Address == address && m.MessageType == MessageType.Write); + if (opRegWriteResponse == null) { - if (snapshot[i].Address == 0x0A && snapshot[i].MessageType == MessageType.Write) - { - opCtrlWriteIdx = i; - break; - } + return new AssertionResult(false, "No response received for OpCtrl write."); } - - if (opCtrlWriteIdx < 0) - return new AssertionResult(false, "DumpEmitsRegisterBurst: no Write reply at OpCtrl (0x0A) found."); - - var coreReads = snapshot + var coreReads = messages .Select((m, i) => (msg: m, idx: i)) - .Where(x => x.msg.Address <= 0x13 && x.msg.MessageType == MessageType.Read) + .Where(x => x.msg.Address <= 32 && x.msg.MessageType == MessageType.Read) .ToList(); - - bool writeBeforeAllReads = coreReads.All(x => opCtrlWriteIdx < x.idx); - if (!writeBeforeAllReads) - return new AssertionResult(false, "DumpEmitsRegisterBurst: OpCtrl Write reply did not precede all core Read replies."); - - var presentAddresses = coreReads.Select(x => (int)x.msg.Address).Distinct().ToHashSet(); - var missing = Enumerable.Range(0, 0x14).Where(a => !presentAddresses.Contains(a)).ToList(); + var uniqueCoreAddresses = coreReads.Select(x => x.msg.Address).Distinct().ToHashSet(); + var missing = Enumerable.Range(0, 18).Where(a => !uniqueCoreAddresses.Contains(a)).ToList(); if (missing.Count > 0) return new AssertionResult(false, - $"DumpEmitsRegisterBurst: missing Read replies for {missing.Count} core address(es): {string.Join(", ", missing.Select(a => $"0x{a:X2}"))}."); + $"Missing Read replies for {missing.Count} core address(es): {string.Join(", ", missing.Select(a => $"0x{a:X2}"))}."); - return new AssertionResult(true, "DumpEmitsRegisterBurst: all 20 core register reads received after OpCtrl write."); + return new AssertionResult(true, "All core register reads received after OpCtrl write."); } catch (Exception ex) { @@ -176,9 +159,7 @@ public async Task DumpEmitsRegisterBurst(string portName) } finally { - subscription?.Dispose(); await Task.Delay(200); - // Ensure we restore original state even though DUMP is transient using (var device = new AsyncDevice(portName)) { From b9b8bdc62c41696b1b58d7047b1347f4b7f69854 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Tue, 23 Jun 2026 04:55:43 -0700 Subject: [PATCH 16/41] Fix hearbeat tests --- .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 106 ++++++++++++++++-- 1 file changed, 98 insertions(+), 8 deletions(-) diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs index 9f54c70..3226e29 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -77,25 +77,24 @@ public async Task VisualEnWritable(string portName) public async Task HeartbeatEnEmitsEvents(string portName) { byte originalOpCtrl = 0; + ushort whoAmI = 0; try { using (var device = new AsyncDevice(portName)) { originalOpCtrl = await device.ReadByteAsync(address); + whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } await Task.Delay(500); // The previous one needs some time to disconnect - var harpDevice = new Bonsai.Harp.Device { PortName = portName }; - var responses = await RegisterHelpers.WriteToTransportAsync( + var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; + var messages = await RegisterHelpers.WriteToTransportAsync( portName, new[] { HarpMessage.FromByte(address, MessageType.Write, 0xE5) }, - TimeSpan.FromSeconds(0.5)); - var messages = await harpDevice.Generate() - .TakeUntil(Observable.Timer(TimeSpan.FromSeconds(2))) - .ToList(); + TimeSpan.FromSeconds(2.0)); - bool received = messages.Any(m => m.Address == 0x18 && m.MessageType == MessageType.Event); + bool received = messages.Any(m => m.Address == 18 && m.MessageType == MessageType.Event); return new AssertionResult( received, @@ -117,10 +116,100 @@ public async Task HeartbeatEnEmitsEvents(string portName) } } + [HarpTest(Description = "Validates that HEARTBEAT_EN (bit 2) takes precedence over ALIVE_EN (bit 7): when both are set, R_HEARTBEAT events are emitted and R_TIMESTAMP_SECOND events are not.")] + public async Task HeartbeatEnPrecedenceOverAliveEn(string portName) + { + byte originalOpCtrl = 0; + ushort whoAmI = 0; + + try + { + using (var device = new AsyncDevice(portName)) + { + originalOpCtrl = await device.ReadByteAsync(address); + whoAmI = await device.ReadUInt16Async(WhoAmI.Address); + } + await Task.Delay(500); + + var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; + // Set both ALIVE_EN (bit 7) and HEARTBEAT_EN (bit 2) with Active mode (bit 0) + var messages = await RegisterHelpers.WriteToTransportAsync( + portName, + new[] { HarpMessage.FromByte(address, MessageType.Write, 0x85) }, + TimeSpan.FromSeconds(2.0)); + + bool receivedHeartbeat = messages.Any(m => m.Address == 18 && m.MessageType == MessageType.Event); + bool receivedTimestamp = messages.Any(m => m.Address == 8 && m.MessageType == MessageType.Event); + + if (!receivedHeartbeat) + return new AssertionResult(false, "HeartbeatEnPrecedenceOverAliveEn: no R_HEARTBEAT event received within 2s (expected HEARTBEAT_EN to take precedence)."); + if (receivedTimestamp) + return new AssertionResult(false, "HeartbeatEnPrecedenceOverAliveEn: R_TIMESTAMP_SECOND event received when HEARTBEAT_EN should suppress it."); + + return new AssertionResult(true, "HeartbeatEnPrecedenceOverAliveEn: R_HEARTBEAT events received and R_TIMESTAMP_SECOND correctly suppressed."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + finally + { + await Task.Delay(200); + using (var device = new AsyncDevice(portName)) + { + await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, originalOpCtrl)); + } + } + } + + [HarpTest(Description = "Validates that ALIVE_EN (deprecated, bit 7) causes R_TIMESTAMP_SECOND events to be emitted when HEARTBEAT_EN is not set.")] + public async Task AliveEnEmitsTimestampEvents(string portName) + { + byte originalOpCtrl = 0; + ushort whoAmI = 0; + + try + { + using (var device = new AsyncDevice(portName)) + { + originalOpCtrl = await device.ReadByteAsync(address); + whoAmI = await device.ReadUInt16Async(WhoAmI.Address); + } + await Task.Delay(500); + + var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; + // Set only ALIVE_EN (bit 7) with Active mode (bit 0); HEARTBEAT_EN (bit 2) is cleared + var messages = await RegisterHelpers.WriteToTransportAsync( + portName, + new[] { HarpMessage.FromByte(address, MessageType.Write, 0x81) }, + TimeSpan.FromSeconds(2.0)); + + bool receivedTimestamp = messages.Any(m => m.Address == 8 && m.MessageType == MessageType.Event); + + if (!receivedTimestamp) + return new Result(false, Status.Skipped, "AliveEnEmitsTimestampEvents: ALIVE_EN is deprecated and R_TIMESTAMP_SECOND events were not emitted."); + + return new AssertionResult(true, "AliveEnEmitsTimestampEvents: R_TIMESTAMP_SECOND event received within 2s."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + finally + { + await Task.Delay(200); + using (var device = new AsyncDevice(portName)) + { + await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, originalOpCtrl)); + } + } + } + [HarpTest(Description = "Validates that the DUMP bit triggers a burst of all core register reads after an OpCtrl write.")] public async Task RegisterDump(string portName) { byte originalOpCtrl = 0; + ushort whoAmI = 0; try { @@ -128,9 +217,10 @@ public async Task RegisterDump(string portName) using (var device = new AsyncDevice(portName)) { originalOpCtrl = await device.ReadByteAsync(address); + whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } - var harpDevice = new Bonsai.Harp.Device { PortName = portName }; + var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; var messages = await RegisterHelpers.WriteToTransportAsync( portName, new[] { HarpMessage.FromByte(address, MessageType.Write, (byte)(originalOpCtrl | 0x08)) }, From 6e651cdef96ab54636442e6875d834b5b7ef7f00 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Tue, 23 Jun 2026 04:56:17 -0700 Subject: [PATCH 17/41] Format --- .../Generate/GenerateRegisterMetadataCommand.cs | 8 ++++---- src/Harp.Toolkit/Generate/GeneratorHelper.cs | 2 +- src/Harp.Toolkit/TaskExtensions.cs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Harp.Toolkit/Generate/GenerateRegisterMetadataCommand.cs b/src/Harp.Toolkit/Generate/GenerateRegisterMetadataCommand.cs index 6c9a0c6..0975834 100644 --- a/src/Harp.Toolkit/Generate/GenerateRegisterMetadataCommand.cs +++ b/src/Harp.Toolkit/Generate/GenerateRegisterMetadataCommand.cs @@ -16,10 +16,10 @@ public GenerateRegisterMetadataCommand() OutputPathOption outputPathOption = new(); Argument registerWorksheetPathArgument = ArgumentValidation.AcceptExistingOnly( new Argument("registers.xls") - { - Description = "The path to the file describing the device registers.", - Arity = ArgumentArity.ExactlyOne - }); + { + Description = "The path to the file describing the device registers.", + Arity = ArgumentArity.ExactlyOne + }); Arguments.Add(registerWorksheetPathArgument); Options.Add(outputPathOption); diff --git a/src/Harp.Toolkit/Generate/GeneratorHelper.cs b/src/Harp.Toolkit/Generate/GeneratorHelper.cs index eeb0a05..a4db8b6 100644 --- a/src/Harp.Toolkit/Generate/GeneratorHelper.cs +++ b/src/Harp.Toolkit/Generate/GeneratorHelper.cs @@ -41,7 +41,7 @@ public static bool AssertNoGeneratorErrors(CompilerErrorCollection errors) Console.Error.WriteLine(errorLog.ToString()); return !errors.HasErrors; } - + return true; } } diff --git a/src/Harp.Toolkit/TaskExtensions.cs b/src/Harp.Toolkit/TaskExtensions.cs index 60668ad..19f2309 100644 --- a/src/Harp.Toolkit/TaskExtensions.cs +++ b/src/Harp.Toolkit/TaskExtensions.cs @@ -1,4 +1,4 @@ -namespace Harp.Toolkit; +namespace Harp.Toolkit; static class TaskExtensions { @@ -13,4 +13,4 @@ internal static async Task WithTimeout(this Task task, int? millisecond } else throw new TimeoutException("There was a timeout while awaiting the device response."); } -} \ No newline at end of file +} From 49a00ff3a083379db7cb30812ea3087f2c0b2709 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:52:39 -0700 Subject: [PATCH 18/41] Add clock alignment and PPS synchronization benchmark tests --- .../Benchmark/BenchmarkCommand.cs | 40 +++++++- .../Benchmark/ClockTestOptions.cs | 18 ++++ .../Benchmark/Suites/ClockTestSuite.cs | 95 +++++++++++++++++++ 3 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 src/Harp.Toolkit/Benchmark/ClockTestOptions.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/ClockTestSuite.cs diff --git a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs b/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs index c2749e7..e4b82a9 100644 --- a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs +++ b/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs @@ -1,5 +1,6 @@ using System.CommandLine; using Spectre.Console; +using Harp.Toolkit.Benchmark; using Harp.Toolkit.Benchmark.Suites; namespace Harp.Toolkit; @@ -20,23 +21,53 @@ public BenchmarkCommand() Description = "Show detailed results for each test.", Required = false, }; + + Option clockPortOption = new("--clock-port") + { + Description = "Serial port of the reference clock device. Enables clock alignment tests.", + Required = false, + }; + + Option regClockOption = new("--pps-address") + { + Description = "Register address on the tested device (--port) that emits an event whenever the incoming PPS signal goes high. Enables PPS alignment test.", + Required = false, + }; + + Option clockSamplesOption = new("--clock-samples") + { + Description = "Number of PPS event pairs to collect for the PPS alignment test. Default: 5.", + Required = false, + }; + clockSamplesOption.DefaultValueFactory = _ => 5; + Options.Add(portNameOption); Options.Add(fileOption); Options.Add(verboseOption); + Options.Add(clockPortOption); + Options.Add(regClockOption); + Options.Add(clockSamplesOption); SetAction(parsedResult => { string portName = parsedResult.GetRequiredValue(portNameOption); FileInfo? reportFile = parsedResult.GetValue(fileOption); bool verbose = parsedResult.GetValue(verboseOption); - return RunBenchmarks(portName, reportFile, verbose, CancellationToken.None); + string? clockPort = parsedResult.GetValue(clockPortOption); + ClockTestOptions? clockOptions = clockPort is null ? null : new ClockTestOptions( + ClockPort: clockPort, + PpsAddress: parsedResult.GetValue(regClockOption), + ClockSamples: parsedResult.GetValue(clockSamplesOption)); + return RunBenchmarks(portName, reportFile, verbose, clockOptions, CancellationToken.None); }); } - static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verbose, CancellationToken cancellationToken) + static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verbose, ClockTestOptions? clockOptions, CancellationToken cancellationToken) { AnsiConsole.MarkupLine($"Running tests on [bold]{portName}[/]..."); + if (clockOptions is not null) + AnsiConsole.MarkupLine($"Clock reference device: [bold]{clockOptions.ClockPort}[/]"); - var runner = new CoreRunner(); + var runner = new CoreRunner(clockOptions); var report = new Report { DeviceName = $"Harp Device ({portName})", @@ -142,7 +173,7 @@ static string GetResultMarkup(IResult result) class CoreRunner : Runner { - public CoreRunner() : base() + public CoreRunner(ClockTestOptions? clockOptions = null) : base() { AddSuite(new R_WHO_AM_I()); AddSuite(new R_HW_VERSION_H()); @@ -165,6 +196,7 @@ public CoreRunner() : base() AddSuite(new R_HEARTBEAT()); AddSuite(new R_VERSION()); AddSuite(new RoundTripTestSuite()); + AddSuite(new ClockTestSuite(clockOptions)); } } } diff --git a/src/Harp.Toolkit/Benchmark/ClockTestOptions.cs b/src/Harp.Toolkit/Benchmark/ClockTestOptions.cs new file mode 100644 index 0000000..0fb6c28 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/ClockTestOptions.cs @@ -0,0 +1,18 @@ +namespace Harp.Toolkit.Benchmark; + +/// +/// Options for clock alignment and PPS synchronization tests run against a reference clock device. +/// +/// +/// Serial port of the reference clock device (WhiteRabbit). Enabling this option runs the +/// simultaneous WhoAmI timestamp comparison test. +/// +/// +/// Register address on the tested device that emits an event whenever the incoming PPS signal +/// goes high. When provided, also runs the PPS alignment test. +/// +/// Number of PPS event pairs to collect for the PPS alignment test. +internal record ClockTestOptions( + string ClockPort, + int? PpsAddress = null, + int ClockSamples = 5); diff --git a/src/Harp.Toolkit/Benchmark/Suites/ClockTestSuite.cs b/src/Harp.Toolkit/Benchmark/Suites/ClockTestSuite.cs new file mode 100644 index 0000000..2172e83 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/ClockTestSuite.cs @@ -0,0 +1,95 @@ + +using Bonsai.Harp; +using Harp.Toolkit.Benchmark; + +namespace Harp.Toolkit.Benchmark.Suites; + +internal class ClockTestSuite : Suite +{ + private const byte OperationControlAddress = 0x0A; + private readonly ClockTestOptions? options; + + public ClockTestSuite(ClockTestOptions? options) + { + this.options = options; + } + + public override string Description => "Tests clock alignment and PPS synchronization accuracy against a reference clock device."; + + [HarpTest(Description = "Sends 100 simultaneous WhoAmI reads to both devices and compares embedded timestamps to measure clock offset.")] + public async Task SimultaneousWhoAmI(string portName) + { + if (options is null) + return new Result(false, Status.Skipped, "No clock port provided (--clock-port)."); + + const int n = 100; + double[] deltas = new double[n]; + var probe = WhoAmI.FromPayload(MessageType.Read, default); + + using var testedDevice = new AsyncDevice(portName); + using var clockDevice = new AsyncDevice(options.ClockPort); + + for (int i = 0; i < n; i++) + { + var results = await Task.WhenAll(testedDevice.CommandAsync(probe), clockDevice.CommandAsync(probe)); + deltas[i] = results[0].GetTimestamp() - results[1].GetTimestamp(); + await Task.Delay(new Random().Next(20, 70)); + } + + var summary = new BenchmarkSummary(deltas); + return new NumericBenchmarkResult( + summary, + Status.Passed, + $"Clock offset: mean={summary.Mean:F6}s ({summary.Mean * 1e3:F3}ms), " + + $"stddev={summary.StdDev:F6}s ({summary.StdDev * 1e3:F3}ms)"); + } + + [HarpTest(Description = "Subscribes to PPS events on both devices and compares timestamps to measure hardware clock synchronization accuracy.")] + public async Task PPSEventAlignment(string portName) + { + if (options is null) + return new Result(false, Status.Skipped, "No clock port provided (--clock-port)."); + if (!options.PpsAddress.HasValue) + return new Result(false, Status.Skipped, "No tested device register provided (--reg-clock)."); + + // The clock device (WhiteRabbit) emits a TimestampSecond event (0x08) on every PPS tick. + // ALIVE_EN (0x80) | OP_MODE (0x01) enables those events. + // TODO: consider also supporting Heartbeat (0x12) via HEARTBEAT_EN (0x04) | OP_MODE (0x01). + const int clockDeviceReg = 0x08; + const byte clockDeviceOpCtrl = 0x81; + + var listenDuration = TimeSpan.FromSeconds(options.ClockSamples + 5); + var allMessages = await Task.WhenAll( + RegisterHelpers.WriteToTransportAsync( + options.ClockPort, + [HarpMessage.FromByte(OperationControlAddress, MessageType.Write, clockDeviceOpCtrl)], + listenDuration), + RegisterHelpers.WriteToTransportAsync( + portName, + [HarpMessage.FromByte(OperationControlAddress, MessageType.Write, 0x01)], + listenDuration)); + + var clockEvents = allMessages[0] + .Where(m => m.MessageType == MessageType.Event && m.Address == clockDeviceReg) + .Take(options.ClockSamples).ToList(); + var testedEvents = allMessages[1] + .Where(m => m.MessageType == MessageType.Event && m.Address == options.PpsAddress!.Value) + .Take(options.ClockSamples).ToList(); + + int pairCount = Math.Min(clockEvents.Count, testedEvents.Count); + if (pairCount == 0) + return new AssertionResult(false, + $"No event pairs received within {listenDuration.TotalSeconds}s " + + $"(clock register 0x{clockDeviceReg:X2}, tested register 0x{options.PpsAddress!.Value:X2})."); + + var deltas = clockEvents + .Zip(testedEvents, (c, t) => c.GetTimestamp() - t.GetTimestamp()) + .ToArray(); + var summary = new BenchmarkSummary(deltas); + return new NumericBenchmarkResult( + summary, + Status.Passed, + $"PPS delta ({pairCount} pairs): mean={summary.Mean:F6}s ({summary.Mean * 1e3:F3}ms), " + + $"stddev={summary.StdDev:F6}s ({summary.StdDev * 1e3:F3}ms)"); + } +} From b7df2d0bcbc39a90dfeb6ecfb9b0823bf76049e1 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:22:19 -0700 Subject: [PATCH 19/41] Add benchmark suite validating registers via generated device.yml interface --- .../Benchmark/BenchmarkCommand.cs | 33 +++- .../Benchmark/ClockTestOptions.cs | 2 +- .../Benchmark/GeneratedInterfaceCompiler.cs | 78 +++++++++ src/Harp.Toolkit/Benchmark/Suite.cs | 38 ++++- .../Benchmark/Suites/DeviceInterfaceSuite.cs | 151 ++++++++++++++++++ src/Harp.Toolkit/Harp.Toolkit.csproj | 2 + 6 files changed, 298 insertions(+), 6 deletions(-) create mode 100644 src/Harp.Toolkit/Benchmark/GeneratedInterfaceCompiler.cs create mode 100644 src/Harp.Toolkit/Benchmark/Suites/DeviceInterfaceSuite.cs diff --git a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs b/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs index e4b82a9..2d4ddef 100644 --- a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs +++ b/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs @@ -1,7 +1,9 @@ using System.CommandLine; using Spectre.Console; +using Harp.Generators; using Harp.Toolkit.Benchmark; using Harp.Toolkit.Benchmark.Suites; +using Harp.Toolkit.Generate; namespace Harp.Toolkit; public class BenchmarkCommand : Command @@ -41,12 +43,20 @@ public BenchmarkCommand() }; clockSamplesOption.DefaultValueFactory = _ => 5; + Option deviceYmlOption = new("--device-yml") + { + Description = "Path to the device's device.yml. Enables validation of the generated C# interface against a live read of every declared register, and cross-checks WhoAmI/firmware/hardware versions.", + Required = false, + }; + OptionValidation.AcceptExistingOnly(deviceYmlOption); + Options.Add(portNameOption); Options.Add(fileOption); Options.Add(verboseOption); Options.Add(clockPortOption); Options.Add(regClockOption); Options.Add(clockSamplesOption); + Options.Add(deviceYmlOption); SetAction(parsedResult => { string portName = parsedResult.GetRequiredValue(portNameOption); @@ -57,17 +67,28 @@ public BenchmarkCommand() ClockPort: clockPort, PpsAddress: parsedResult.GetValue(regClockOption), ClockSamples: parsedResult.GetValue(clockSamplesOption)); - return RunBenchmarks(portName, reportFile, verbose, clockOptions, CancellationToken.None); + FileInfo? deviceYml = parsedResult.GetValue(deviceYmlOption); + return RunBenchmarks(portName, reportFile, verbose, clockOptions, deviceYml, CancellationToken.None); }); } - static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verbose, ClockTestOptions? clockOptions, CancellationToken cancellationToken) + static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verbose, ClockTestOptions? clockOptions, FileInfo? deviceYml, CancellationToken cancellationToken) { AnsiConsole.MarkupLine($"Running tests on [bold]{portName}[/]..."); if (clockOptions is not null) AnsiConsole.MarkupLine($"Clock reference device: [bold]{clockOptions.ClockPort}[/]"); - var runner = new CoreRunner(clockOptions); + DeviceMetadata? deviceMetadata = null; + string? deviceRawYaml = null; + if (deviceYml is not null) + { + AnsiConsole.Markup($"Loading device metadata from [bold]{deviceYml.FullName}[/]..."); + deviceMetadata = GeneratorHelper.ReadDeviceMetadata(deviceYml.FullName); + deviceRawYaml = await File.ReadAllTextAsync(deviceYml.FullName, cancellationToken); + AnsiConsole.MarkupLine($" [green]Done![/] ({deviceMetadata.Registers.Count} registers)"); + } + + var runner = new CoreRunner(clockOptions, deviceMetadata, deviceRawYaml); var report = new Report { DeviceName = $"Harp Device ({portName})", @@ -173,7 +194,10 @@ static string GetResultMarkup(IResult result) class CoreRunner : Runner { - public CoreRunner(ClockTestOptions? clockOptions = null) : base() + public CoreRunner( + ClockTestOptions? clockOptions = null, + DeviceMetadata? deviceMetadata = null, + string? deviceRawYaml = null) : base() { AddSuite(new R_WHO_AM_I()); AddSuite(new R_HW_VERSION_H()); @@ -197,6 +221,7 @@ public CoreRunner(ClockTestOptions? clockOptions = null) : base() AddSuite(new R_VERSION()); AddSuite(new RoundTripTestSuite()); AddSuite(new ClockTestSuite(clockOptions)); + AddSuite(new DeviceInterfaceSuite(deviceMetadata, deviceRawYaml)); } } } diff --git a/src/Harp.Toolkit/Benchmark/ClockTestOptions.cs b/src/Harp.Toolkit/Benchmark/ClockTestOptions.cs index 0fb6c28..875f4cd 100644 --- a/src/Harp.Toolkit/Benchmark/ClockTestOptions.cs +++ b/src/Harp.Toolkit/Benchmark/ClockTestOptions.cs @@ -1,4 +1,4 @@ -namespace Harp.Toolkit.Benchmark; +namespace Harp.Toolkit.Benchmark; /// /// Options for clock alignment and PPS synchronization tests run against a reference clock device. diff --git a/src/Harp.Toolkit/Benchmark/GeneratedInterfaceCompiler.cs b/src/Harp.Toolkit/Benchmark/GeneratedInterfaceCompiler.cs new file mode 100644 index 0000000..63fe9d5 --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/GeneratedInterfaceCompiler.cs @@ -0,0 +1,78 @@ +using System.Reflection; +using System.Text; +using Harp.Generators; +using Harp.Toolkit.Generate; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Emit; +using Microsoft.Extensions.DependencyModel; + +namespace Harp.Toolkit.Benchmark; + +/// +/// Generates the C# interface for a device.yml (via ), +/// compiles it in-memory, and returns its register address-to-type map so callers can +/// invoke each register's own generated parser reflectively. +/// +internal static class GeneratedInterfaceCompiler +{ + public static IReadOnlyDictionary Compile(DeviceMetadata deviceOnlyMetadata, string rawDeviceYaml, string @namespace) + { + var generator = new InterfaceGenerator(deviceOnlyMetadata, @namespace); + var implementation = generator.GenerateImplementation(); + if (!GeneratorHelper.AssertNoGeneratorErrors(generator.Errors)) + throw new InvalidOperationException("Interface generation from device.yml completed with errors."); + + var syntaxTree = CSharpSyntaxTree.ParseText(implementation.Device); + var compilation = CSharpCompilation.Create( + $"HarpGeneratedInterface_{@namespace}", + new[] { syntaxTree }, + GetMetadataReferences(), + new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, allowUnsafe: true)); + + // The generated Device class's static constructor reads device.yml back from + // an embedded "{Namespace}.device.yml" manifest resource (e.g. to expose it via + // the Metadata property) - without it, merely accessing RegisterMap throws. + var rawYamlBytes = Encoding.UTF8.GetBytes(rawDeviceYaml); + var deviceYamlResource = new ResourceDescription( + $"{@namespace}.device.yml", + () => new MemoryStream(rawYamlBytes), + isPublic: true); + + using var peStream = new MemoryStream(); + var result = compilation.Emit(peStream, manifestResources: new[] { deviceYamlResource }); + if (!result.Success) + { + var errors = string.Join(Environment.NewLine, result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error)); + throw new InvalidOperationException($"Failed to compile the interface generated from device.yml:{Environment.NewLine}{errors}"); + } + + var assembly = Assembly.Load(peStream.ToArray()); + var deviceType = assembly.GetType($"{@namespace}.Device") + ?? throw new InvalidOperationException($"Compiled assembly does not contain type '{@namespace}.Device'."); + var registerMapProperty = deviceType.GetProperty("RegisterMap", BindingFlags.Public | BindingFlags.Static) + ?? throw new InvalidOperationException($"'{@namespace}.Device' does not expose a static RegisterMap property."); + + return (IReadOnlyDictionary)registerMapProperty.GetValue(null)!; + } + + // Reuses this project's existing PreserveCompilationContext setup (already required + // for RazorLight's own runtime compilation) to resolve the full reference-assembly + // closure, including Bonsai.Harp/Bonsai.Core, which the generated code depends on. + private static IReadOnlyList GetMetadataReferences() + { + var context = DependencyContext.Default + ?? throw new InvalidOperationException("No DependencyContext available for compiling the generated interface."); + + var paths = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var library in context.CompileLibraries) + { + foreach (var path in library.ResolveReferencePaths()) + { + paths.Add(path); + } + } + + return paths.Select(path => (MetadataReference)MetadataReference.CreateFromFile(path)).ToList(); + } +} diff --git a/src/Harp.Toolkit/Benchmark/Suite.cs b/src/Harp.Toolkit/Benchmark/Suite.cs index 3df8108..1278b68 100644 --- a/src/Harp.Toolkit/Benchmark/Suite.cs +++ b/src/Harp.Toolkit/Benchmark/Suite.cs @@ -9,7 +9,14 @@ public abstract class Suite { public abstract string Description { get; } - public int TestCount => CollectTests().Count(); + /// + /// Tests whose number and identity is only known at runtime. + /// During test collection, these will be enumerated and run + /// after the fixed methods. + /// + protected virtual IReadOnlyList DynamicTests { get; } = new List(); + + public int TestCount => CollectTests().Count() + DynamicTests.Count; private IEnumerable<(MethodInfo Method, HarpTestAttribute Attribute)> CollectTests() { @@ -56,9 +63,38 @@ public async IAsyncEnumerable RunAllAsync(string portName, [Enumer Description = attr.Description ?? string.Empty }; } + + foreach (var test in DynamicTests) + { + cancellationToken.ThrowIfCancellationRequested(); + + onTestStart?.Invoke(test.Name, test.Description); + + IResult testResult; + try + { + testResult = await test.Run(portName, cancellationToken); + } + catch (Exception ex) + { + testResult = new ErrorResult(ex); + } + yield return new MethodResult + { + Result = testResult, + Name = test.Name, + Description = test.Description + }; + } } } +/// +/// A test whose name and behavior is determined at runtime rather than declared +/// with on a fixed method. +/// +public record DynamicTest(string Name, string Description, Func> Run); + public class SuiteResult { public required string Name { get; set; } diff --git a/src/Harp.Toolkit/Benchmark/Suites/DeviceInterfaceSuite.cs b/src/Harp.Toolkit/Benchmark/Suites/DeviceInterfaceSuite.cs new file mode 100644 index 0000000..3fbe08a --- /dev/null +++ b/src/Harp.Toolkit/Benchmark/Suites/DeviceInterfaceSuite.cs @@ -0,0 +1,151 @@ +using System.Reflection; +using Bonsai.Harp; +using Harp.Generators; +using Harp.Toolkit.Benchmark; + +namespace Harp.Toolkit.Benchmark.Suites; + +/// +/// Validates a live device against the C# interface actually generated from device.yml, +/// not just the schema: for every register, it reads the live reply and parses it with +/// that register's own generated GetPayload(HarpMessage) (found via the generated +/// , address -> register type, rather than +/// by guessing method names), catching real codegen/parser bugs (bad bit offsets, wrong +/// enum casts, mismatched payload struct fields) that a schema-only structural check +/// can't. It also cross-checks the WhoAmI/firmware/hardware version reported by the +/// device against device.yml. Core registers (WhoAmI, Heartbeat, version registers, etc.) +/// are covered automatically via the base +/// chain that generated code links into, so this only needs to generate/compile +/// device-specific registers. +/// +/// +/// Generating/compiling the interface and the identity check are always exactly one +/// test each, so they're plain methods like every other +/// suite. Only the per-register checks are s, since the +/// register set is only known once device.yml has been parsed. The compile itself +/// happens eagerly in the constructor (not inside ) +/// so the resulting register map is available up front to build those dynamic tests. +/// +internal class DeviceInterfaceSuite : Suite +{ + private readonly DeviceMetadata? metadata; + private readonly IReadOnlyDictionary? registerMap; + private readonly Exception? compileError; + private readonly IReadOnlyList dynamicTests; + + public DeviceInterfaceSuite(DeviceMetadata? metadata, string? rawYaml) + { + this.metadata = metadata; + + if (metadata is not null && rawYaml is not null) + { + try + { + registerMap = GeneratedInterfaceCompiler.Compile(metadata, rawYaml, $"Harp.{metadata.Device}"); + } + catch (Exception ex) + { + while (ex is TargetInvocationException or TypeInitializationException && ex.InnerException is not null) + ex = ex.InnerException; + compileError = ex; + } + } + + dynamicTests = registerMap is null + ? new List() + : BuildRegisterTests(registerMap); + } + + protected override IReadOnlyList DynamicTests => dynamicTests; + + public override string Description => + "Validates the C# interface generated from device.yml by parsing live register replies with its own generated parsers, and cross-checks WhoAmI/firmware/hardware versions."; + + [HarpTest(Description = "Generates and compiles the C# interface from device.yml.")] + public Task GenerateAndCompileInterface(string portName) + { + IResult result = metadata is null + ? new Result(false, Status.Skipped, "No device.yml provided (--device-yml).") + : registerMap is not null + ? new AssertionResult(true, $"Generated and compiled the interface with {registerMap.Count} registers.") + : new ErrorResult(compileError!); + return Task.FromResult(result); + } + + [HarpTest(Description = "Compares the WhoAmI/firmware/hardware version reported by the device against device.yml.")] + public async Task DeviceIdentity(string portName) + { + if (metadata is null) + return new Result(false, Status.Skipped, "No device.yml provided (--device-yml)."); + + using var device = new AsyncDevice(portName); + var mismatches = new List(); + + int whoAmI = await device.ReadWhoAmIAsync(); + if (whoAmI != metadata.WhoAmI) + mismatches.Add($"WhoAmI: device={whoAmI}, device.yml={metadata.WhoAmI}"); + + // HarpVersion.Satisfies treats a null Major/Minor on the argument as a wildcard, + // so a device.yml that only pins a major version (minor left unset) is honored + // automatically - no need to hand-roll that comparison. + var firmware = await device.ReadFirmwareVersionAsync(); + if (metadata.FirmwareVersion is not null && !firmware.Satisfies(metadata.FirmwareVersion)) + mismatches.Add($"Firmware version: device={firmware}, device.yml={metadata.FirmwareVersion}"); + + var hardware = await device.ReadHardwareVersionAsync(); + if (metadata.HardwareTargets is not null && !hardware.Satisfies(metadata.HardwareTargets)) + mismatches.Add($"Hardware version: device={hardware}, device.yml={metadata.HardwareTargets}"); + + return new AssertionResult( + mismatches.Count == 0, + _ => mismatches.Count == 0 + ? $"WhoAmI={whoAmI}, Firmware={firmware}, Hardware={hardware} match device.yml." + : string.Join("; ", mismatches)); + } + + private static IReadOnlyList BuildRegisterTests(IReadOnlyDictionary registerMap) + { + return registerMap + .Where(entry => entry.Value.IsPublic) + .OrderBy(entry => entry.Key) + .Select(entry => new DynamicTest( + entry.Value.Name, + $"Reads register '{entry.Value.Name}' (address {entry.Key}) and parses the reply with its generated GetPayload parser.", + (portName, cancellationToken) => CheckRegisterAsync(entry.Key, entry.Value, portName, cancellationToken))) + .ToList(); + } + + private static async Task CheckRegisterAsync(int address, Type registerType, string portName, CancellationToken cancellationToken) + { + const BindingFlags staticMembers = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static; + + var getPayload = registerType.GetMethod("GetPayload", staticMembers, null, new[] { typeof(HarpMessage) }, null); + if (getPayload is null) + return new Result(false, Status.Skipped, $"Generated register '{registerType.Name}' has no GetPayload(HarpMessage) parser."); + + var payloadType = (PayloadType)registerType.GetField("RegisterType", staticMembers)!.GetValue(null)!; + + using var device = new AsyncDevice(portName); + HarpMessage reply; + try + { + reply = await device.CommandAsync(HarpCommand.Read(address, payloadType), cancellationToken); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + + try + { + var value = getPayload.Invoke(null, new object?[] { reply }); + return new AssertionResult(true, $"Register '{registerType.Name}' parsed successfully: {value}."); + } + catch (TargetInvocationException ex) + { + // Unwrap so a real generated-parser bug (bad bit offset, wrong enum cast, ...) + // is reported distinctly from the reflection-invocation exception itself. + return new ErrorResult(ex.InnerException ?? ex); + } + } +} diff --git a/src/Harp.Toolkit/Harp.Toolkit.csproj b/src/Harp.Toolkit/Harp.Toolkit.csproj index 2c2931f..9d6a864 100644 --- a/src/Harp.Toolkit/Harp.Toolkit.csproj +++ b/src/Harp.Toolkit/Harp.Toolkit.csproj @@ -13,6 +13,8 @@ + + From 406c5d9c2723b06258015ec92d510ecaea364afe Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 1 Sep 2026 21:18:29 +0100 Subject: [PATCH 20/41] Rename the benchmark command to verify The command, its folder, namespace and class are renamed from benchmark to verify, so the name describes conformance against the Harp specification rather than just the smaller part that measures timing. The measurement types keep the benchmark name, since NumericBenchmarkResult, BenchmarkSummary and BenchmarkRoundTrip do measure latency. --- src/Harp.Toolkit/Harp.Toolkit.csproj | 2 +- src/Harp.Toolkit/Program.cs | 2 +- .../{Benchmark => Verify}/ClockTestOptions.cs | 2 +- .../GeneratedInterfaceCompiler.cs | 2 +- .../{Benchmark => Verify}/HarpTestAttribute.cs | 0 .../{Benchmark => Verify}/HtmlReportGenerator.cs | 2 +- src/Harp.Toolkit/{Benchmark => Verify}/Report.cs | 0 .../{Benchmark => Verify}/ReportTemplate.cshtml | 0 src/Harp.Toolkit/{Benchmark => Verify}/Result.cs | 0 src/Harp.Toolkit/{Benchmark => Verify}/Runner.cs | 0 src/Harp.Toolkit/{Benchmark => Verify}/Suite.cs | 0 .../{Benchmark => Verify}/Suites/ClockTestSuite.cs | 4 ++-- .../Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs | 2 +- .../Suites/CoreRegisters/R_CLOCK_CONFIG.cs | 2 +- .../Suites/CoreRegisters/R_CORE_VERSION_H.cs | 2 +- .../Suites/CoreRegisters/R_CORE_VERSION_L.cs | 2 +- .../Suites/CoreRegisters/R_DEVICE_NAME.cs | 2 +- .../Suites/CoreRegisters/R_FW_VERSION_H.cs | 2 +- .../Suites/CoreRegisters/R_FW_VERSION_L.cs | 2 +- .../Suites/CoreRegisters/R_HEARTBEAT.cs | 2 +- .../Suites/CoreRegisters/R_HW_VERSION_H.cs | 2 +- .../Suites/CoreRegisters/R_HW_VERSION_L.cs | 2 +- .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 2 +- .../Suites/CoreRegisters/R_RESET_DEV.cs | 2 +- .../Suites/CoreRegisters/R_SERIAL_NUMBER.cs | 2 +- .../Suites/CoreRegisters/R_TAG.cs | 2 +- .../Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs | 2 +- .../Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs | 2 +- .../Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs | 2 +- .../Suites/CoreRegisters/R_UID.cs | 2 +- .../Suites/CoreRegisters/R_VERSION.cs | 2 +- .../Suites/CoreRegisters/R_WHO_AM_I.cs | 2 +- .../Suites/CoreRegisters/_RegisterHelpers.cs | 2 +- .../Suites/DeviceInterfaceSuite.cs | 4 ++-- .../Suites/RoundTripTestSuite.cs | 2 +- .../VerifyCommand.cs} | 14 +++++++------- 36 files changed, 38 insertions(+), 38 deletions(-) rename src/Harp.Toolkit/{Benchmark => Verify}/ClockTestOptions.cs (95%) rename src/Harp.Toolkit/{Benchmark => Verify}/GeneratedInterfaceCompiler.cs (99%) rename src/Harp.Toolkit/{Benchmark => Verify}/HarpTestAttribute.cs (100%) rename src/Harp.Toolkit/{Benchmark => Verify}/HtmlReportGenerator.cs (89%) rename src/Harp.Toolkit/{Benchmark => Verify}/Report.cs (100%) rename src/Harp.Toolkit/{Benchmark => Verify}/ReportTemplate.cshtml (100%) rename src/Harp.Toolkit/{Benchmark => Verify}/Result.cs (100%) rename src/Harp.Toolkit/{Benchmark => Verify}/Runner.cs (100%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suite.cs (100%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/ClockTestSuite.cs (98%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs (94%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_CLOCK_CONFIG.cs (97%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_CORE_VERSION_H.cs (95%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_CORE_VERSION_L.cs (95%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_DEVICE_NAME.cs (96%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_FW_VERSION_H.cs (95%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_FW_VERSION_L.cs (95%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_HEARTBEAT.cs (96%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_HW_VERSION_H.cs (95%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_HW_VERSION_L.cs (95%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_OPERATION_CTRL.cs (99%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_RESET_DEV.cs (92%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_SERIAL_NUMBER.cs (96%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_TAG.cs (97%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs (97%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs (97%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs (98%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_UID.cs (97%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_VERSION.cs (97%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/R_WHO_AM_I.cs (96%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/CoreRegisters/_RegisterHelpers.cs (98%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/DeviceInterfaceSuite.cs (99%) rename src/Harp.Toolkit/{Benchmark => Verify}/Suites/RoundTripTestSuite.cs (97%) rename src/Harp.Toolkit/{Benchmark/BenchmarkCommand.cs => Verify/VerifyCommand.cs} (94%) diff --git a/src/Harp.Toolkit/Harp.Toolkit.csproj b/src/Harp.Toolkit/Harp.Toolkit.csproj index 9d6a864..571acd5 100644 --- a/src/Harp.Toolkit/Harp.Toolkit.csproj +++ b/src/Harp.Toolkit/Harp.Toolkit.csproj @@ -22,7 +22,7 @@ - + PreserveNewest diff --git a/src/Harp.Toolkit/Program.cs b/src/Harp.Toolkit/Program.cs index de2dbfe..87076d9 100644 --- a/src/Harp.Toolkit/Program.cs +++ b/src/Harp.Toolkit/Program.cs @@ -16,7 +16,7 @@ static async Task Main(string[] args) rootCommand.Subcommands.Add(new ListCommand()); rootCommand.Subcommands.Add(new UpdateFirmwareCommand()); rootCommand.Subcommands.Add(new GenerateCommand()); - rootCommand.Subcommands.Add(new BenchmarkCommand()); + rootCommand.Subcommands.Add(new VerifyCommand()); rootCommand.SetAction(async parseResult => { var portName = parseResult.GetRequiredValue(portNameOption); diff --git a/src/Harp.Toolkit/Benchmark/ClockTestOptions.cs b/src/Harp.Toolkit/Verify/ClockTestOptions.cs similarity index 95% rename from src/Harp.Toolkit/Benchmark/ClockTestOptions.cs rename to src/Harp.Toolkit/Verify/ClockTestOptions.cs index 875f4cd..96b867e 100644 --- a/src/Harp.Toolkit/Benchmark/ClockTestOptions.cs +++ b/src/Harp.Toolkit/Verify/ClockTestOptions.cs @@ -1,4 +1,4 @@ -namespace Harp.Toolkit.Benchmark; +namespace Harp.Toolkit.Verify; /// /// Options for clock alignment and PPS synchronization tests run against a reference clock device. diff --git a/src/Harp.Toolkit/Benchmark/GeneratedInterfaceCompiler.cs b/src/Harp.Toolkit/Verify/GeneratedInterfaceCompiler.cs similarity index 99% rename from src/Harp.Toolkit/Benchmark/GeneratedInterfaceCompiler.cs rename to src/Harp.Toolkit/Verify/GeneratedInterfaceCompiler.cs index 63fe9d5..cf23c1b 100644 --- a/src/Harp.Toolkit/Benchmark/GeneratedInterfaceCompiler.cs +++ b/src/Harp.Toolkit/Verify/GeneratedInterfaceCompiler.cs @@ -7,7 +7,7 @@ using Microsoft.CodeAnalysis.Emit; using Microsoft.Extensions.DependencyModel; -namespace Harp.Toolkit.Benchmark; +namespace Harp.Toolkit.Verify; /// /// Generates the C# interface for a device.yml (via ), diff --git a/src/Harp.Toolkit/Benchmark/HarpTestAttribute.cs b/src/Harp.Toolkit/Verify/HarpTestAttribute.cs similarity index 100% rename from src/Harp.Toolkit/Benchmark/HarpTestAttribute.cs rename to src/Harp.Toolkit/Verify/HarpTestAttribute.cs diff --git a/src/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs b/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs similarity index 89% rename from src/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs rename to src/Harp.Toolkit/Verify/HtmlReportGenerator.cs index ede8e25..396afd8 100644 --- a/src/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs +++ b/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs @@ -14,7 +14,7 @@ public static async Task GenerateAsync(Report report) // The template is copied to the output directory under Reporting/ReportTemplate.cshtml // RazorLight expects the path relative to the project root (which we set to the assembly location) - string templatePath = Path.Combine("Benchmark", "ReportTemplate.cshtml"); + string templatePath = Path.Combine("Verify", "ReportTemplate.cshtml"); return await engine.CompileRenderAsync(templatePath, report); } diff --git a/src/Harp.Toolkit/Benchmark/Report.cs b/src/Harp.Toolkit/Verify/Report.cs similarity index 100% rename from src/Harp.Toolkit/Benchmark/Report.cs rename to src/Harp.Toolkit/Verify/Report.cs diff --git a/src/Harp.Toolkit/Benchmark/ReportTemplate.cshtml b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml similarity index 100% rename from src/Harp.Toolkit/Benchmark/ReportTemplate.cshtml rename to src/Harp.Toolkit/Verify/ReportTemplate.cshtml diff --git a/src/Harp.Toolkit/Benchmark/Result.cs b/src/Harp.Toolkit/Verify/Result.cs similarity index 100% rename from src/Harp.Toolkit/Benchmark/Result.cs rename to src/Harp.Toolkit/Verify/Result.cs diff --git a/src/Harp.Toolkit/Benchmark/Runner.cs b/src/Harp.Toolkit/Verify/Runner.cs similarity index 100% rename from src/Harp.Toolkit/Benchmark/Runner.cs rename to src/Harp.Toolkit/Verify/Runner.cs diff --git a/src/Harp.Toolkit/Benchmark/Suite.cs b/src/Harp.Toolkit/Verify/Suite.cs similarity index 100% rename from src/Harp.Toolkit/Benchmark/Suite.cs rename to src/Harp.Toolkit/Verify/Suite.cs diff --git a/src/Harp.Toolkit/Benchmark/Suites/ClockTestSuite.cs b/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs similarity index 98% rename from src/Harp.Toolkit/Benchmark/Suites/ClockTestSuite.cs rename to src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs index 2172e83..cfeaa4c 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/ClockTestSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs @@ -1,8 +1,8 @@  using Bonsai.Harp; -using Harp.Toolkit.Benchmark; +using Harp.Toolkit.Verify; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class ClockTestSuite : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs similarity index 94% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs index 0089263..e64e69c 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs @@ -1,6 +1,6 @@  using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_ASSEMBLY_VERSION : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs similarity index 97% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs index 50b044e..ff2c40b 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CLOCK_CONFIG.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs @@ -1,7 +1,7 @@ using System.Text; using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_CLOCK_CONFIG : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs similarity index 95% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs index 6bc733a..6b103d2 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_CORE_VERSION_H : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs similarity index 95% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs index 7be1036..32f66dd 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_CORE_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_CORE_VERSION_L : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs similarity index 96% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs index f0d516c..27721f6 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_DEVICE_NAME.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_DEVICE_NAME : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs similarity index 95% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs index 0808222..1b82ecf 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_FW_VERSION_H : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs similarity index 95% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs index 637e7a1..6cf2daf 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_FW_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_FW_VERSION_L : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs similarity index 96% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs index ffb3e4c..452c23e 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HEARTBEAT.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_HEARTBEAT : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs similarity index 95% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs index 41bff61..d2b1778 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_HW_VERSION_H : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs similarity index 95% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs index fdd328c..450964f 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_HW_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_HW_VERSION_L : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs similarity index 99% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs index 3226e29..e5164e8 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -2,7 +2,7 @@ using System.Reactive.Linq; using System.Collections.Concurrent; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_OPERATION_CTRL : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs similarity index 92% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs index 4e80a83..5daac7d 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_RESET_DEV.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_RESET_DEV : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_SERIAL_NUMBER.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs similarity index 96% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_SERIAL_NUMBER.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs index 65a475b..7d7846e 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_SERIAL_NUMBER.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs @@ -1,6 +1,6 @@  using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_SERIAL_NUMBER : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs similarity index 97% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs index 7406bfe..96674cb 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TAG.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_TAG : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs similarity index 97% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs index 3fdc73f..0ac2700 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_TIMESTAMP_MICRO : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs similarity index 97% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs index e67623c..95dfded 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs @@ -1,6 +1,6 @@  using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_TIMESTAMP_OFFSET : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs similarity index 98% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs index 489be6d..2a3ba30 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs @@ -1,7 +1,7 @@  using Bonsai.Harp; using System.Diagnostics; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_TIMESTAMP_SECOND : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_UID.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs similarity index 97% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_UID.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs index 31a7fee..33e0de9 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_UID.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs @@ -1,6 +1,6 @@  using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_UID : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs similarity index 97% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs index e914bf4..65db8b4 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_VERSION.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs @@ -1,6 +1,6 @@ using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_VERSION : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs similarity index 96% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs index 368fae3..defd0c2 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/R_WHO_AM_I.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs @@ -1,6 +1,6 @@  using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_WHO_AM_I : Suite { diff --git a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/_RegisterHelpers.cs similarity index 98% rename from src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/_RegisterHelpers.cs index dfb82f8..2d9113c 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/CoreRegisters/_RegisterHelpers.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/_RegisterHelpers.cs @@ -3,7 +3,7 @@ using System.Reactive.Linq; using System.Reactive.Subjects; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal static class RegisterHelpers { diff --git a/src/Harp.Toolkit/Benchmark/Suites/DeviceInterfaceSuite.cs b/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs similarity index 99% rename from src/Harp.Toolkit/Benchmark/Suites/DeviceInterfaceSuite.cs rename to src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs index 3fbe08a..acee796 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/DeviceInterfaceSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs @@ -1,9 +1,9 @@ using System.Reflection; using Bonsai.Harp; using Harp.Generators; -using Harp.Toolkit.Benchmark; +using Harp.Toolkit.Verify; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; /// /// Validates a live device against the C# interface actually generated from device.yml, diff --git a/src/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs b/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs similarity index 97% rename from src/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs rename to src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs index 840b112..27c0c2a 100644 --- a/src/Harp.Toolkit/Benchmark/Suites/RoundTripTestSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs @@ -1,6 +1,6 @@  using Bonsai.Harp; -namespace Harp.Toolkit.Benchmark.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class RoundTripTestSuite : Suite { diff --git a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs similarity index 94% rename from src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs rename to src/Harp.Toolkit/Verify/VerifyCommand.cs index 2d4ddef..f8c1580 100644 --- a/src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -1,15 +1,15 @@ using System.CommandLine; using Spectre.Console; using Harp.Generators; -using Harp.Toolkit.Benchmark; -using Harp.Toolkit.Benchmark.Suites; +using Harp.Toolkit.Verify; +using Harp.Toolkit.Verify.Suites; using Harp.Toolkit.Generate; namespace Harp.Toolkit; -public class BenchmarkCommand : Command +public class VerifyCommand : Command { - public BenchmarkCommand() - : base("benchmark", "Run benchmark tests on the device.") + public VerifyCommand() + : base("verify", "Verify device conformance against the Harp specification.") { PortNameOption portNameOption = new(); Option fileOption = new("--report") @@ -68,11 +68,11 @@ public BenchmarkCommand() PpsAddress: parsedResult.GetValue(regClockOption), ClockSamples: parsedResult.GetValue(clockSamplesOption)); FileInfo? deviceYml = parsedResult.GetValue(deviceYmlOption); - return RunBenchmarks(portName, reportFile, verbose, clockOptions, deviceYml, CancellationToken.None); + return RunVerification(portName, reportFile, verbose, clockOptions, deviceYml, CancellationToken.None); }); } - static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verbose, ClockTestOptions? clockOptions, FileInfo? deviceYml, CancellationToken cancellationToken) + static async Task RunVerification(string portName, FileInfo? reportFile, bool verbose, ClockTestOptions? clockOptions, FileInfo? deviceYml, CancellationToken cancellationToken) { AnsiConsole.MarkupLine($"Running tests on [bold]{portName}[/]..."); if (clockOptions is not null) From 40010f059c254e1ae0808a51327e7422d0cf86c0 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 2 Sep 2026 09:21:24 +0100 Subject: [PATCH 21/41] Clean up register access and naming in verify Replaces the literal register addresses and lengths across the suites with the constants declared by the client register classes. The four addresses the client cannot name are written in decimal to match the numbering used by the specification and by core.yml, and the UID address becomes internal so the serial number suite reads it instead of repeating the value. R_VERSION has no client constant either, so it gains Version, VersionPayload and SemanticVersion, mirroring the interface the code generator produces for a register carrying a payload specification. Each version field is read as three bytes, major, minor and patch, rather than as HarpVersion, which has no patch component. The Version suite gains a test reporting the parsed PROTOCOL, FIRMWARE, HARDWARE, CORE_ID and INTERFACE_HASH fields. Renames the PPS option to --pps-event, replacing a partial rename that left --pps-address in the option and --reg-clock in the skip message, so the skip message named a flag the command rejected. The same file also drops a conditional access on a non-nullable result property, which was the only build warning on the branch. --- src/Harp.Toolkit/Verify/ClockTestOptions.cs | 8 +- .../Verify/Suites/ClockTestSuite.cs | 21 ++- .../CoreRegisters/R_ASSEMBLY_VERSION.cs | 4 +- .../Suites/CoreRegisters/R_CLOCK_CONFIG.cs | 5 +- .../Suites/CoreRegisters/R_CORE_VERSION_H.cs | 5 +- .../Suites/CoreRegisters/R_CORE_VERSION_L.cs | 5 +- .../Suites/CoreRegisters/R_DEVICE_NAME.cs | 6 +- .../Suites/CoreRegisters/R_FW_VERSION_H.cs | 5 +- .../Suites/CoreRegisters/R_FW_VERSION_L.cs | 5 +- .../Suites/CoreRegisters/R_HEARTBEAT.cs | 6 +- .../Suites/CoreRegisters/R_HW_VERSION_H.cs | 5 +- .../Suites/CoreRegisters/R_HW_VERSION_L.cs | 5 +- .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 45 +++-- .../Suites/CoreRegisters/R_RESET_DEV.cs | 3 +- .../Suites/CoreRegisters/R_SERIAL_NUMBER.cs | 10 +- .../Verify/Suites/CoreRegisters/R_TAG.cs | 10 +- .../Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs | 7 +- .../CoreRegisters/R_TIMESTAMP_OFFSET.cs | 10 +- .../CoreRegisters/R_TIMESTAMP_SECOND.cs | 2 +- .../Verify/Suites/CoreRegisters/R_UID.cs | 18 +- .../Verify/Suites/CoreRegisters/R_VERSION.cs | 39 ++++- .../Verify/Suites/CoreRegisters/R_WHO_AM_I.cs | 2 +- ..._RegisterHelpers.cs => RegisterHelpers.cs} | 0 .../Verify/Suites/CoreRegisters/Version.cs | 164 ++++++++++++++++++ src/Harp.Toolkit/Verify/VerifyCommand.cs | 12 +- 25 files changed, 291 insertions(+), 111 deletions(-) rename src/Harp.Toolkit/Verify/Suites/CoreRegisters/{_RegisterHelpers.cs => RegisterHelpers.cs} (100%) create mode 100644 src/Harp.Toolkit/Verify/Suites/CoreRegisters/Version.cs diff --git a/src/Harp.Toolkit/Verify/ClockTestOptions.cs b/src/Harp.Toolkit/Verify/ClockTestOptions.cs index 96b867e..0f1fd19 100644 --- a/src/Harp.Toolkit/Verify/ClockTestOptions.cs +++ b/src/Harp.Toolkit/Verify/ClockTestOptions.cs @@ -7,12 +7,12 @@ /// Serial port of the reference clock device (WhiteRabbit). Enabling this option runs the /// simultaneous WhoAmI timestamp comparison test. /// -/// -/// Register address on the tested device that emits an event whenever the incoming PPS signal -/// goes high. When provided, also runs the PPS alignment test. +/// +/// Address of the register on the tested device that reports the incoming PPS pulse from the +/// reference clock device. When provided, also runs the PPS alignment test. /// /// Number of PPS event pairs to collect for the PPS alignment test. internal record ClockTestOptions( string ClockPort, - int? PpsAddress = null, + int? PpsEvent = null, int ClockSamples = 5); diff --git a/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs b/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs index cfeaa4c..adb216f 100644 --- a/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs @@ -6,7 +6,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class ClockTestSuite : Suite { - private const byte OperationControlAddress = 0x0A; private readonly ClockTestOptions? options; public ClockTestSuite(ClockTestOptions? options) @@ -45,42 +44,42 @@ public async Task SimultaneousWhoAmI(string portName) } [HarpTest(Description = "Subscribes to PPS events on both devices and compares timestamps to measure hardware clock synchronization accuracy.")] - public async Task PPSEventAlignment(string portName) + public async Task PpsEventAlignment(string portName) { if (options is null) return new Result(false, Status.Skipped, "No clock port provided (--clock-port)."); - if (!options.PpsAddress.HasValue) - return new Result(false, Status.Skipped, "No tested device register provided (--reg-clock)."); + if (!options.PpsEvent.HasValue) + return new Result(false, Status.Skipped, "No tested device register provided (--pps-event)."); - // The clock device (WhiteRabbit) emits a TimestampSecond event (0x08) on every PPS tick. + // The clock device (WhiteRabbit) emits a TimestampSecond event on every PPS tick. // ALIVE_EN (0x80) | OP_MODE (0x01) enables those events. - // TODO: consider also supporting Heartbeat (0x12) via HEARTBEAT_EN (0x04) | OP_MODE (0x01). - const int clockDeviceReg = 0x08; + // TODO: consider also supporting Heartbeat via HEARTBEAT_EN (0x04) | OP_MODE (0x01). + const int clockDeviceReg = TimestampSeconds.Address; const byte clockDeviceOpCtrl = 0x81; var listenDuration = TimeSpan.FromSeconds(options.ClockSamples + 5); var allMessages = await Task.WhenAll( RegisterHelpers.WriteToTransportAsync( options.ClockPort, - [HarpMessage.FromByte(OperationControlAddress, MessageType.Write, clockDeviceOpCtrl)], + [HarpMessage.FromByte(OperationControl.Address, MessageType.Write, clockDeviceOpCtrl)], listenDuration), RegisterHelpers.WriteToTransportAsync( portName, - [HarpMessage.FromByte(OperationControlAddress, MessageType.Write, 0x01)], + [HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x01)], listenDuration)); var clockEvents = allMessages[0] .Where(m => m.MessageType == MessageType.Event && m.Address == clockDeviceReg) .Take(options.ClockSamples).ToList(); var testedEvents = allMessages[1] - .Where(m => m.MessageType == MessageType.Event && m.Address == options.PpsAddress!.Value) + .Where(m => m.MessageType == MessageType.Event && m.Address == options.PpsEvent!.Value) .Take(options.ClockSamples).ToList(); int pairCount = Math.Min(clockEvents.Count, testedEvents.Count); if (pairCount == 0) return new AssertionResult(false, $"No event pairs received within {listenDuration.TotalSeconds}s " + - $"(clock register 0x{clockDeviceReg:X2}, tested register 0x{options.PpsAddress!.Value:X2})."); + $"(clock register {clockDeviceReg}, tested register {options.PpsEvent!.Value})."); var deltas = clockEvents .Zip(testedEvents, (c, t) => c.GetTimestamp() - t.GetTimestamp()) diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs index e64e69c..df28c24 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs @@ -15,8 +15,8 @@ public async Task AssertReturnsZero(string portName) return new AssertionResult( value == 0x00, x => x ? - $"AssemblyVersion register correctly returned 0x00." : - $"AssemblyVersion register returned a non-zero value (0x{value:X2})"); + "AssemblyVersion register correctly returned 0x00." : + $"AssemblyVersion register returned a non-zero value (0x{value:X2})."); } } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs index ff2c40b..c411c2a 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs @@ -5,7 +5,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_CLOCK_CONFIG : Suite { - private const byte address = 0x0E; public override string Description => "Clock Configuration Register Tests"; [HarpTest(Description = "Validates that ClockConfig register is readable.")] @@ -13,7 +12,7 @@ public async Task IsReadable(string portName) { using (var device = new AsyncDevice(portName)) { - return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), address, "ClockConfig"); + return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), ClockConfiguration.Address, "ClockConfig"); } } @@ -22,7 +21,7 @@ public async Task ReportSyncCapability(string portName) { using (var device = new AsyncDevice(portName)) { - var value = await device.ReadByteAsync(address); + var value = await device.ReadByteAsync(ClockConfiguration.Address); bool repAble = (value & (1 << 3)) != 0; bool genAble = (value & (1 << 4)) != 0; StringBuilder sb = new StringBuilder("ClockConfig sync capability:"); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs index 6b103d2..f706ce0 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs @@ -4,7 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_CORE_VERSION_H : Suite { - private const byte address = 0x04; public override string Description => "Core Version High Register Tests"; [HarpTest(Description = "Validates that CoreVersionHigh matches byte 0 of R_VERSION.")] @@ -12,8 +11,8 @@ public async Task AssertConsistentWithVersion(string portName) { using (var device = new AsyncDevice(portName)) { - var versionArray = await device.ReadByteArrayAsync(0x13); - var registerValue = await device.ReadByteAsync(address); + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(CoreVersionHigh.Address); return new AssertionResult( registerValue == versionArray[0], x => x diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs index 32f66dd..a7e526e 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs @@ -4,7 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_CORE_VERSION_L : Suite { - private const byte address = 0x05; public override string Description => "Core Version Low Register Tests"; [HarpTest(Description = "Validates that CoreVersionLow matches byte 1 of R_VERSION.")] @@ -12,8 +11,8 @@ public async Task AssertConsistentWithVersion(string portName) { using (var device = new AsyncDevice(portName)) { - var versionArray = await device.ReadByteArrayAsync(0x13); - var registerValue = await device.ReadByteAsync(address); + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(CoreVersionLow.Address); return new AssertionResult( registerValue == versionArray[1], x => x diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs index 27721f6..cf63fe7 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs @@ -4,8 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_DEVICE_NAME : Suite { - private const byte address = 0x0C; - private const int expectedLength = 25; public override string Description => "Device Name Register Tests"; [HarpTest(Description = "Validates that DeviceName register is readable.")] @@ -15,7 +13,7 @@ public async Task IsReadable(string portName) { try { - await device.ReadByteArrayAsync(address); + await device.ReadByteArrayAsync(DeviceName.Address); return new AssertionResult(true, "DeviceName is readable."); } catch (Exception ex) @@ -30,7 +28,7 @@ public async Task AssertLength(string portName) { using (var device = new AsyncDevice(portName)) { - return await RegisterHelpers.AssertReadableArrayAsync(device, address, expectedLength, "DeviceName"); + return await RegisterHelpers.AssertReadableArrayAsync(device, DeviceName.Address, DeviceName.RegisterLength, "DeviceName"); } } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs index 1b82ecf..616486e 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs @@ -4,7 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_FW_VERSION_H : Suite { - private const byte address = 0x06; public override string Description => "Firmware Version High Register Tests"; [HarpTest(Description = "Validates that FwVersionHigh matches byte 3 of R_VERSION.")] @@ -12,8 +11,8 @@ public async Task AssertConsistentWithVersion(string portName) { using (var device = new AsyncDevice(portName)) { - var versionArray = await device.ReadByteArrayAsync(0x13); - var registerValue = await device.ReadByteAsync(address); + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(FirmwareVersionHigh.Address); return new AssertionResult( registerValue == versionArray[3], x => x diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs index 6cf2daf..b2d95a1 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs @@ -4,7 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_FW_VERSION_L : Suite { - private const byte address = 0x07; public override string Description => "Firmware Version Low Register Tests"; [HarpTest(Description = "Validates that FwVersionLow matches byte 4 of R_VERSION.")] @@ -12,8 +11,8 @@ public async Task AssertConsistentWithVersion(string portName) { using (var device = new AsyncDevice(portName)) { - var versionArray = await device.ReadByteArrayAsync(0x13); - var registerValue = await device.ReadByteAsync(address); + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(FirmwareVersionLow.Address); return new AssertionResult( registerValue == versionArray[4], x => x diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs index 452c23e..01f9515 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs @@ -4,7 +4,7 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_HEARTBEAT : Suite { - private const byte address = 18; + private const byte Address = 18; public override string Description => "Heartbeat Register Tests"; [HarpTest(Description = "Validates that Heartbeat register is readable.")] @@ -12,7 +12,7 @@ public async Task IsReadable(string portName) { using (var device = new AsyncDevice(portName)) { - return await RegisterHelpers.AssertReadableAsync(a => device.ReadUInt16Async(a), address, "Heartbeat"); + return await RegisterHelpers.AssertReadableAsync(a => device.ReadUInt16Async(a), Address, "Heartbeat"); } } @@ -21,7 +21,7 @@ public async Task IsNotWritable(string portName) { using (var device = new AsyncDevice(portName)) { - var req = HarpMessage.FromByte(address, MessageType.Write, 0x00); + var req = HarpMessage.FromByte(Address, MessageType.Write, 0x00); var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( rejected, diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs index d2b1778..efe1009 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs @@ -4,7 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_HW_VERSION_H : Suite { - private const byte address = 0x01; public override string Description => "Hardware Version High Register Tests"; [HarpTest(Description = "Validates that HwVersionHigh matches byte 6 of R_VERSION.")] @@ -12,8 +11,8 @@ public async Task AssertConsistentWithVersion(string portName) { using (var device = new AsyncDevice(portName)) { - var versionArray = await device.ReadByteArrayAsync(0x13); - var registerValue = await device.ReadByteAsync(address); + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(HardwareVersionHigh.Address); return new AssertionResult( registerValue == versionArray[6], x => x diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs index 450964f..9781a52 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs @@ -4,7 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_HW_VERSION_L : Suite { - private const byte address = 0x02; public override string Description => "Hardware Version Low Register Tests"; [HarpTest(Description = "Validates that HwVersionLow matches byte 7 of R_VERSION.")] @@ -12,8 +11,8 @@ public async Task AssertConsistentWithVersion(string portName) { using (var device = new AsyncDevice(portName)) { - var versionArray = await device.ReadByteArrayAsync(0x13); - var registerValue = await device.ReadByteAsync(address); + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(HardwareVersionLow.Address); return new AssertionResult( registerValue == versionArray[7], x => x diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs index e5164e8..dc5708b 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -1,4 +1,4 @@ -using Bonsai.Harp; +using Bonsai.Harp; using System.Reactive.Linq; using System.Collections.Concurrent; @@ -6,7 +6,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_OPERATION_CTRL : Suite { - private const byte address = 0x0A; public override string Description => "Operation Control Register Tests"; [HarpTest(Description = "Validates that OP_MODE bits can be round-tripped between Standby (0) and Active (1).")] @@ -14,15 +13,15 @@ public async Task OpModeRoundTrip(string portName) { using (var device = new AsyncDevice(portName)) { - var original = await device.ReadByteAsync(address); + var original = await device.ReadByteAsync(OperationControl.Address); byte currentMode = (byte)(original & 0x03); byte newMode = currentMode == 0x01 ? (byte)0x00 : (byte)0x01; byte newValue = (byte)((original & ~0x03) | newMode); try { - await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, newValue)); - var readBack = await device.ReadByteAsync(address); + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, newValue)); + var readBack = await device.ReadByteAsync(OperationControl.Address); byte readMode = (byte)(readBack & 0x03); return new AssertionResult( @@ -36,7 +35,7 @@ public async Task OpModeRoundTrip(string portName) // Always restore original state try { - await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, original)); + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, original)); } catch { @@ -83,7 +82,7 @@ public async Task HeartbeatEnEmitsEvents(string portName) { using (var device = new AsyncDevice(portName)) { - originalOpCtrl = await device.ReadByteAsync(address); + originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } await Task.Delay(500); // The previous one needs some time to disconnect @@ -91,7 +90,7 @@ public async Task HeartbeatEnEmitsEvents(string portName) var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; var messages = await RegisterHelpers.WriteToTransportAsync( portName, - new[] { HarpMessage.FromByte(address, MessageType.Write, 0xE5) }, + new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0xE5) }, TimeSpan.FromSeconds(2.0)); bool received = messages.Any(m => m.Address == 18 && m.MessageType == MessageType.Event); @@ -111,7 +110,7 @@ public async Task HeartbeatEnEmitsEvents(string portName) await Task.Delay(200); // Wait for port to be released before reopening using (var device = new AsyncDevice(portName)) { - await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, originalOpCtrl)); + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, originalOpCtrl)); } } } @@ -126,7 +125,7 @@ public async Task HeartbeatEnPrecedenceOverAliveEn(string portName) { using (var device = new AsyncDevice(portName)) { - originalOpCtrl = await device.ReadByteAsync(address); + originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } await Task.Delay(500); @@ -135,7 +134,7 @@ public async Task HeartbeatEnPrecedenceOverAliveEn(string portName) // Set both ALIVE_EN (bit 7) and HEARTBEAT_EN (bit 2) with Active mode (bit 0) var messages = await RegisterHelpers.WriteToTransportAsync( portName, - new[] { HarpMessage.FromByte(address, MessageType.Write, 0x85) }, + new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x85) }, TimeSpan.FromSeconds(2.0)); bool receivedHeartbeat = messages.Any(m => m.Address == 18 && m.MessageType == MessageType.Event); @@ -157,7 +156,7 @@ public async Task HeartbeatEnPrecedenceOverAliveEn(string portName) await Task.Delay(200); using (var device = new AsyncDevice(portName)) { - await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, originalOpCtrl)); + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, originalOpCtrl)); } } } @@ -172,7 +171,7 @@ public async Task AliveEnEmitsTimestampEvents(string portName) { using (var device = new AsyncDevice(portName)) { - originalOpCtrl = await device.ReadByteAsync(address); + originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } await Task.Delay(500); @@ -181,7 +180,7 @@ public async Task AliveEnEmitsTimestampEvents(string portName) // Set only ALIVE_EN (bit 7) with Active mode (bit 0); HEARTBEAT_EN (bit 2) is cleared var messages = await RegisterHelpers.WriteToTransportAsync( portName, - new[] { HarpMessage.FromByte(address, MessageType.Write, 0x81) }, + new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x81) }, TimeSpan.FromSeconds(2.0)); bool receivedTimestamp = messages.Any(m => m.Address == 8 && m.MessageType == MessageType.Event); @@ -200,7 +199,7 @@ public async Task AliveEnEmitsTimestampEvents(string portName) await Task.Delay(200); using (var device = new AsyncDevice(portName)) { - await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, originalOpCtrl)); + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, originalOpCtrl)); } } } @@ -216,17 +215,17 @@ public async Task RegisterDump(string portName) // Read original state before modifying using (var device = new AsyncDevice(portName)) { - originalOpCtrl = await device.ReadByteAsync(address); + originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; var messages = await RegisterHelpers.WriteToTransportAsync( portName, - new[] { HarpMessage.FromByte(address, MessageType.Write, (byte)(originalOpCtrl | 0x08)) }, + new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, (byte)(originalOpCtrl | 0x08)) }, TimeSpan.FromSeconds(1)); - var opRegWriteResponse = messages.FirstOrDefault(m => m.Address == address && m.MessageType == MessageType.Write); + var opRegWriteResponse = messages.FirstOrDefault(m => m.Address == OperationControl.Address && m.MessageType == MessageType.Write); if (opRegWriteResponse == null) { return new AssertionResult(false, "No response received for OpCtrl write."); @@ -253,21 +252,21 @@ public async Task RegisterDump(string portName) // Ensure we restore original state even though DUMP is transient using (var device = new AsyncDevice(portName)) { - await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, originalOpCtrl)); + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, originalOpCtrl)); } } } private static async Task TestOptionalBitAsync(AsyncDevice device, string bitName, byte bitMask) { - var original = await device.ReadByteAsync(address); + var original = await device.ReadByteAsync(OperationControl.Address); byte toggled = (byte)(original ^ bitMask); try { try { - await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, toggled)); + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, toggled)); } catch (HarpException) { @@ -275,7 +274,7 @@ private static async Task TestOptionalBitAsync(AsyncDevice device, stri $"{bitName} is optional/deprecated and not supported by this device."); } - var readBack = await device.ReadByteAsync(address); + var readBack = await device.ReadByteAsync(OperationControl.Address); bool bitChanged = (readBack & bitMask) == (toggled & bitMask); return new AssertionResult( @@ -288,7 +287,7 @@ private static async Task TestOptionalBitAsync(AsyncDevice device, stri { try { - await device.CommandAsync(HarpMessage.FromByte(address, MessageType.Write, original)); + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, original)); } catch { diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs index 5daac7d..405d11d 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs @@ -4,7 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_RESET_DEV : Suite { - private const byte address = 0x0B; public override string Description => "Reset Device Register Tests"; [HarpTest(Description = "Validates that ResetDev register is readable.")] @@ -12,7 +11,7 @@ public async Task IsReadable(string portName) { using (var device = new AsyncDevice(portName)) { - return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), address, "ResetDev"); + return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), ResetDevice.Address, "ResetDev"); } } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs index 7d7846e..a398840 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs @@ -6,14 +6,14 @@ internal class R_SERIAL_NUMBER : Suite { public override string Description => "Serial Number Register Tests"; - [HarpTest(Description = "Validates the contents of the register match the lower two bytes of R_UID")] - public async Task AssertConsitentWithUid(string portName) + [HarpTest(Description = "Validates that SerialNumber matches the first two bytes of R_UID.")] + public async Task AssertConsistentWithUid(string portName) { using (var device = new AsyncDevice(portName)) { - var uidValue = await device.ReadByteArrayAsync(0x10); + var uidValue = await device.ReadByteArrayAsync(R_UID.Address); if (uidValue.Length < 2) - throw new ArgumentException($"Expected UID register contents to be at least 2 bytes. Got {uidValue.Length}"); + throw new ArgumentException($"Expected UID register contents to be at least 2 bytes. Got {uidValue.Length}."); var twoFirstBytes = BitConverter.ToInt16(uidValue, 0); var serialNumberValue = await device.ReadSerialNumberAsync(); @@ -21,7 +21,7 @@ public async Task AssertConsitentWithUid(string portName) return new AssertionResult( twoFirstBytes == serialNumberValue, x => x ? - $"SerialNumber register contents are consistent with UID register." : + "SerialNumber register contents are consistent with UID register." : $"SerialNumber register content (0x{serialNumberValue:X4}) does not match the first two bytes of UID register (0x{twoFirstBytes:X4})."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs index 96674cb..b615f2a 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs @@ -4,8 +4,8 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_TAG : Suite { - private const byte address = 0x11; - private const int expectedLength = 8; + private const byte Address = 17; + private const int ExpectedLength = 8; public override string Description => "Tag Register Tests"; [HarpTest(Description = "Validates that Tag register is readable.")] @@ -15,7 +15,7 @@ public async Task IsReadable(string portName) { try { - await device.ReadByteArrayAsync(address); + await device.ReadByteArrayAsync(Address); return new AssertionResult(true, "Tag is readable."); } catch (Exception ex) @@ -30,7 +30,7 @@ public async Task AssertLength(string portName) { using (var device = new AsyncDevice(portName)) { - return await RegisterHelpers.AssertReadableArrayAsync(device, address, expectedLength, "Tag"); + return await RegisterHelpers.AssertReadableArrayAsync(device, Address, ExpectedLength, "Tag"); } } @@ -39,7 +39,7 @@ public async Task IsNotWritable(string portName) { using (var device = new AsyncDevice(portName)) { - var req = HarpMessage.FromByte(address, MessageType.Write, 0x00); + var req = HarpMessage.FromByte(Address, MessageType.Write, 0x00); var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( rejected, diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs index 0ac2700..beaacd6 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs @@ -4,7 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_TIMESTAMP_MICRO : Suite { - private const byte address = 0x09; public override string Description => "Timestamp Microseconds Register Tests"; [HarpTest(Description = "Validates that TimestampMicro register is readable.")] @@ -14,7 +13,7 @@ public async Task IsReadable(string portName) { try { - await device.ReadUInt16Async(address); + await device.ReadUInt16Async(TimestampMicroseconds.Address); return new AssertionResult(true, "TimestampMicro is readable."); } catch (Exception ex) @@ -29,7 +28,7 @@ public async Task IsNotWritable(string portName) { using (var device = new AsyncDevice(portName)) { - var req = HarpMessage.FromUInt16(address, MessageType.Write, 0); + var req = HarpMessage.FromUInt16(TimestampMicroseconds.Address, MessageType.Write, 0); var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( rejected, @@ -44,7 +43,7 @@ public async Task ValueWithinBounds(string portName) { using (var device = new AsyncDevice(portName)) { - var microValue = await device.ReadUInt16Async(address); + var microValue = await device.ReadUInt16Async(TimestampMicroseconds.Address); return new AssertionResult( microValue < 31250, x => x diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs index 95dfded..c73a198 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs @@ -4,7 +4,7 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_TIMESTAMP_OFFSET : Suite { - private const byte address = 0x0F; + private const byte Address = 15; public override string Description => "Timestamp Offset Register Tests"; [HarpTest(Description = "Validates the deprecated register TimestampOffset returns 0x00.")] @@ -12,12 +12,12 @@ public async Task AssertReturnsZero(string portName) { using (var device = new AsyncDevice(portName)) { - var value = await device.ReadByteAsync(address); + var value = await device.ReadByteAsync(Address); return new AssertionResult( value == 0x00, x => x ? - $"TimestampOffset register correctly returned 0x00." : - $"TimestampOffset register returned a non-zero value (0x{value:X2})"); + "TimestampOffset register correctly returned 0x00." : + $"TimestampOffset register returned a non-zero value (0x{value:X2})."); } } @@ -26,7 +26,7 @@ public async Task IsNotWritable(string portName) { using (var device = new AsyncDevice(portName)) { - var req = HarpMessage.FromByte(address, MessageType.Write, 0x00); + var req = HarpMessage.FromByte(Address, MessageType.Write, 0x00); var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( rejected, diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs index 2a3ba30..9738f47 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs @@ -19,7 +19,7 @@ public async Task IsWritable(string portName) double readSeconds = response.GetTimestamp(); return new AssertionResult( readSeconds - setSeconds < 1.0, - (success) => success ? $"`TimestampSeconds` register is writable and updates as expected." : $"`TimestampSeconds` register is not writable, Expected value: {setSeconds}, read value: {readSeconds}."); + (success) => success ? "TimestampSeconds register is writable and updates as expected." : $"TimestampSeconds register is not writable. Expected value: {setSeconds}, read value: {readSeconds}."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs index 33e0de9..7fc7fb0 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs @@ -4,21 +4,21 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_UID : Suite { - private const byte address = 0x10; - private const byte expected_length = 16; + internal const byte Address = 16; + private const byte ExpectedLength = 16; public override string Description => "UID Register Tests"; - [HarpTest(Description = "Validates whether the UID register is 0 and thus likely not in use.")] + [HarpTest(Description = "Validates that UID register has exactly 16 bytes.")] public async Task AssertLength(string portName) { using (var device = new AsyncDevice(portName)) { - var value = await device.ReadByteArrayAsync(address); + var value = await device.ReadByteArrayAsync(Address); return new AssertionResult( - value.Length == expected_length, + value.Length == ExpectedLength, x => x ? - $"Length is 16 as expected." : - $"Expected length of register to be 16, got {value.Length} instead"); + $"Length is {ExpectedLength} as expected." : + $"Expected length of register to be {ExpectedLength}, got {value.Length} instead."); } } @@ -27,8 +27,8 @@ public async Task AssertReturnsZero(string portName) { using (var device = new AsyncDevice(portName)) { - var value = await device.ReadByteArrayAsync(address); - string msg = value.All(x => x == 0) ? "Value of all bytes is 0. Register likely not being used" : $"Register returned a non-zero value: {BitConverter.ToString(value)}"; + var value = await device.ReadByteArrayAsync(Address); + string msg = value.All(x => x == 0) ? "Value of all bytes is 0. Register likely not being used." : $"Register returned a non-zero value: {BitConverter.ToString(value)}."; return new Result( value, Status.Passed, diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs index 65db8b4..7bfcb89 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs @@ -4,8 +4,6 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_VERSION : Suite { - private const byte address = 0x13; - private const int expectedLength = 32; public override string Description => "Version Register Tests"; [HarpTest(Description = "Validates that Version register is readable.")] @@ -15,7 +13,7 @@ public async Task IsReadable(string portName) { try { - await device.ReadByteArrayAsync(address); + await device.ReadByteArrayAsync(Version.Address); return new AssertionResult(true, "Version is readable."); } catch (Exception ex) @@ -30,7 +28,7 @@ public async Task AssertLength(string portName) { using (var device = new AsyncDevice(portName)) { - return await RegisterHelpers.AssertReadableArrayAsync(device, address, expectedLength, "Version"); + return await RegisterHelpers.AssertReadableArrayAsync(device, Version.Address, Version.RegisterLength, "Version"); } } @@ -39,7 +37,7 @@ public async Task IsNotWritable(string portName) { using (var device = new AsyncDevice(portName)) { - var req = HarpMessage.FromByte(address, MessageType.Write, 0x00); + var req = HarpMessage.FromByte(Version.Address, MessageType.Write, 0x00); var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( rejected, @@ -48,4 +46,35 @@ public async Task IsNotWritable(string portName) : "Version register should NOT be writable."); } } + + [HarpTest(Description = "Reports the version information declared by the device.")] + public async Task ReportVersionInformation(string portName) + { + using (var device = new AsyncDevice(portName)) + { + try + { + var reply = await device.CommandAsync(HarpCommand.ReadByte(Version.Address)); + var payload = reply.GetPayloadArray(); + if (payload.Length != Version.RegisterLength) + { + return new AssertionResult( + false, + $"Version returned {payload.Length} bytes, expected {Version.RegisterLength}."); + } + + var version = Version.GetPayload(reply); + return new Result( + version, + Status.Passed, + $"PROTOCOL {version.ProtocolVersion}, FIRMWARE {version.FirmwareVersion}, " + + $"HARDWARE {version.HardwareVersion}, CORE_ID {version.CoreId}, " + + $"INTERFACE_HASH {Convert.ToHexString(version.InterfaceHash)}."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + } + } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs index defd0c2..851376c 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs @@ -24,7 +24,7 @@ public async Task IsNotWritable(string portName) { using (var device = new AsyncDevice(portName)) { - var req = HarpMessage.FromUInt16(0x00, MessageType.Write, 0); + var req = HarpMessage.FromUInt16(WhoAmI.Address, MessageType.Write, 0); var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( rejected, diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/_RegisterHelpers.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/RegisterHelpers.cs similarity index 100% rename from src/Harp.Toolkit/Verify/Suites/CoreRegisters/_RegisterHelpers.cs rename to src/Harp.Toolkit/Verify/Suites/CoreRegisters/RegisterHelpers.cs diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/Version.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/Version.cs new file mode 100644 index 0000000..3681d03 --- /dev/null +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/Version.cs @@ -0,0 +1,164 @@ +using System.Text; +using Bonsai.Harp; + +namespace Harp.Toolkit.Verify.Suites; + +/// +/// Provides methods for manipulating messages from the Version register. +/// +internal partial class Version +{ + /// + /// Represents the address of the register. This field is constant. + /// + public const int Address = 19; + + /// + /// Represents the payload type of the register. This field is constant. + /// + public const PayloadType RegisterType = PayloadType.U8; + + /// + /// Represents the length of the register. This field is constant. + /// + public const int RegisterLength = 32; + + static VersionPayload ParsePayload(byte[] payload) + { + VersionPayload result; + result.ProtocolVersion = ReadSemanticVersion(new ArraySegment(payload, 0, 3)); + result.FirmwareVersion = ReadSemanticVersion(new ArraySegment(payload, 3, 3)); + result.HardwareVersion = ReadSemanticVersion(new ArraySegment(payload, 6, 3)); + result.CoreId = ReadUtf8String(new ArraySegment(payload, 9, 3)); + result.InterfaceHash = GetSubArray(payload, 12, 20); + return result; + } + + /// + /// Returns the payload data for register messages. + /// + /// A object representing the register message. + /// A value representing the message payload. + public static VersionPayload GetPayload(HarpMessage message) + { + return ParsePayload(message.GetPayloadArray()); + } + + static SemanticVersion ReadSemanticVersion(ArraySegment segment) + { + var array = segment.Array!; + return new SemanticVersion( + array[segment.Offset], + array[segment.Offset + 1], + array[segment.Offset + 2]); + } + + static string ReadUtf8String(ArraySegment segment) + { + var array = segment.Array!; + var count = Array.IndexOf(array, (byte)0, segment.Offset, segment.Count) - segment.Offset; + return Encoding.UTF8.GetString(array, segment.Offset, count < 0 ? segment.Count : count); + } + + static byte[] GetSubArray(byte[] array, int offset, int count) + { + var result = new byte[count]; + Array.Copy(array, offset, result, 0, count); + return result; + } +} + +/// +/// Represents the payload of the Version register. +/// +internal struct VersionPayload +{ + /// + /// The semantic version of the Harp protocol implemented by the device. + /// + public SemanticVersion ProtocolVersion; + + /// + /// The semantic version of the device firmware application. + /// + public SemanticVersion FirmwareVersion; + + /// + /// The semantic version of the device hardware. + /// + public SemanticVersion HardwareVersion; + + /// + /// The three-character code of the Harp microcontroller core targeted by the device + /// firmware. + /// + public string CoreId; + + /// + /// The SHA-1 hash value of the device interface schema file, all zeros when the device + /// declares no schema for the controller to validate against. + /// + public byte[] InterfaceHash; + + /// + /// Returns a that represents the payload of the Version register. + /// + /// + /// A that represents the payload of the Version register. + /// + public override readonly string ToString() + { + return "VersionPayload { " + + "ProtocolVersion = " + ProtocolVersion + ", " + + "FirmwareVersion = " + FirmwareVersion + ", " + + "HardwareVersion = " + HardwareVersion + ", " + + "CoreId = " + CoreId + ", " + + "InterfaceHash = " + (InterfaceHash is null ? string.Empty : Convert.ToHexString(InterfaceHash)) + " " + + "}"; + } +} + +/// +/// Represents the semantic version of a device component reported by the Version register. +/// +internal readonly struct SemanticVersion +{ + /// + /// Initializes a new instance of the structure. + /// + /// The major version number. + /// The minor version number. + /// The patch version number. + public SemanticVersion(byte major, byte minor, byte patch) + { + Major = major; + Minor = minor; + Patch = patch; + } + + /// + /// The major version number. + /// + public byte Major { get; } + + /// + /// The minor version number. + /// + public byte Minor { get; } + + /// + /// The patch version number. + /// + public byte Patch { get; } + + /// + /// Returns a that represents the semantic version. + /// + /// + /// A containing the major, minor and patch numbers separated by dots. + /// + public override string ToString() + { + return $"{Major}.{Minor}.{Patch}"; + } +} diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index f8c1580..4649530 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -30,9 +30,9 @@ public VerifyCommand() Required = false, }; - Option regClockOption = new("--pps-address") + Option ppsEventOption = new("--pps-event") { - Description = "Register address on the tested device (--port) that emits an event whenever the incoming PPS signal goes high. Enables PPS alignment test.", + Description = "Address of the register on the tested device (--port) that reports the incoming PPS pulse from the reference clock device. Enables the PPS alignment test.", Required = false, }; @@ -54,7 +54,7 @@ public VerifyCommand() Options.Add(fileOption); Options.Add(verboseOption); Options.Add(clockPortOption); - Options.Add(regClockOption); + Options.Add(ppsEventOption); Options.Add(clockSamplesOption); Options.Add(deviceYmlOption); SetAction(parsedResult => @@ -65,7 +65,7 @@ public VerifyCommand() string? clockPort = parsedResult.GetValue(clockPortOption); ClockTestOptions? clockOptions = clockPort is null ? null : new ClockTestOptions( ClockPort: clockPort, - PpsAddress: parsedResult.GetValue(regClockOption), + PpsEvent: parsedResult.GetValue(ppsEventOption), ClockSamples: parsedResult.GetValue(clockSamplesOption)); FileInfo? deviceYml = parsedResult.GetValue(deviceYmlOption); return RunVerification(portName, reportFile, verbose, clockOptions, deviceYml, CancellationToken.None); @@ -142,7 +142,7 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve if (test.Result is NumericBenchmarkResult bsr) { - details = $"Mean: {bsr.Summary.Mean:F4}\nMedian: {bsr.Summary.Median:F4}\nStdDev: {bsr.Summary.StdDev:F4}\nMin: {bsr.Summary.Min:F4}\nMax: {bsr.Summary.Max:F4}\nPercentiles: 99th={bsr.Summary.Percentile99:F4}, 01th={bsr.Summary.Percentile01:F4}"; + details = $"Mean: {bsr.Summary.Mean:F4}\nMedian: {bsr.Summary.Median:F4}\nStdDev: {bsr.Summary.StdDev:F4}\nMin: {bsr.Summary.Min:F4}\nMax: {bsr.Summary.Max:F4}\nPercentiles: 99th={bsr.Summary.Percentile99:F4}, 1st={bsr.Summary.Percentile01:F4}"; } else if (test.Result is ErrorResult er) { @@ -150,7 +150,7 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve } else { - var valProp = test.Result?.GetType().GetProperty("Value"); + var valProp = test.Result.GetType().GetProperty("Value"); if (valProp != null) { var val = valProp.GetValue(test.Result); From 824a682c88314a1528a64f7fad39821fbdaf60a0 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 2 Sep 2026 09:55:52 +0100 Subject: [PATCH 22/41] Skip console progress output when redirected The progress line and the line clear that precedes each result are now written only when output is not redirected. Piping the command to a file or running it under CI would otherwise fail on Console.WindowWidth, which is unavailable in that case. Redirected output now carries only the result lines. --- src/Harp.Toolkit/Verify/VerifyCommand.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index 4649530..7897118 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -100,11 +100,13 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve { // Print "Running" status before test execution (without newline) currentTest++; - Console.Write($"({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{testName} .... Running..."); + if (!Console.IsOutputRedirected) + Console.Write($"({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{testName} .... Running..."); })) { // Clear the line by moving cursor to start and overwriting with spaces, then print result - Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r"); + if (!Console.IsOutputRedirected) + Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r"); AnsiConsole.MarkupLine($"[grey]({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}"); var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name); From 53fd1307cdae83e9809581df002c17a555b7e7d2 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 2 Sep 2026 11:14:27 +0100 Subject: [PATCH 23/41] Fix core register test payloads and coverage The read-only probes for R_HEARTBEAT, R_TAG and R_VERSION now send a payload matching the register type and length. The timestamp writability test compares the absolute difference, so a device that ignores the write no longer passes when its clock reads below the value written. The register dump requires all twenty allocated core addresses rather than the first eighteen, and its core boundary now excludes address 32, the first application register. The three operation control event tests now write distinct bit combinations: heartbeat only, alive only, and both. The heartbeat test previously also set ALIVE_EN and the indicator bits. Restores are wrapped so a failed restore no longer replaces the test result. On the command, --clock-samples is rejected when not positive, since zero yielded NaN statistics, and the unreachable report filename fallback is removed. --- .../Verify/HtmlReportGenerator.cs | 2 +- .../Verify/Suites/ClockTestSuite.cs | 2 +- .../Suites/CoreRegisters/R_HEARTBEAT.cs | 4 +- .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 100 +++++++----------- .../Verify/Suites/CoreRegisters/R_TAG.cs | 2 +- .../CoreRegisters/R_TIMESTAMP_SECOND.cs | 2 +- .../Verify/Suites/CoreRegisters/R_VERSION.cs | 2 +- src/Harp.Toolkit/Verify/VerifyCommand.cs | 11 +- 8 files changed, 56 insertions(+), 69 deletions(-) diff --git a/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs b/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs index 396afd8..5686c34 100644 --- a/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs +++ b/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs @@ -12,7 +12,7 @@ public static async Task GenerateAsync(Report report) .UseMemoryCachingProvider() .Build(); - // The template is copied to the output directory under Reporting/ReportTemplate.cshtml + // The template is copied to the output directory under Verify/ReportTemplate.cshtml // RazorLight expects the path relative to the project root (which we set to the assembly location) string templatePath = Path.Combine("Verify", "ReportTemplate.cshtml"); diff --git a/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs b/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs index adb216f..5ea922e 100644 --- a/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs @@ -32,7 +32,7 @@ public async Task SimultaneousWhoAmI(string portName) { var results = await Task.WhenAll(testedDevice.CommandAsync(probe), clockDevice.CommandAsync(probe)); deltas[i] = results[0].GetTimestamp() - results[1].GetTimestamp(); - await Task.Delay(new Random().Next(20, 70)); + await Task.Delay(Random.Shared.Next(20, 70)); } var summary = new BenchmarkSummary(deltas); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs index 01f9515..7fa7f47 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs @@ -4,7 +4,7 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_HEARTBEAT : Suite { - private const byte Address = 18; + internal const byte Address = 18; public override string Description => "Heartbeat Register Tests"; [HarpTest(Description = "Validates that Heartbeat register is readable.")] @@ -21,7 +21,7 @@ public async Task IsNotWritable(string portName) { using (var device = new AsyncDevice(portName)) { - var req = HarpMessage.FromByte(Address, MessageType.Write, 0x00); + var req = HarpMessage.FromUInt16(Address, MessageType.Write, 0); var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( rejected, diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs index dc5708b..a04cdba 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -1,11 +1,11 @@ using Bonsai.Harp; -using System.Reactive.Linq; -using System.Collections.Concurrent; namespace Harp.Toolkit.Verify.Suites; internal class R_OPERATION_CTRL : Suite { + private const int PortReleaseDelayMilliseconds = 200; + public override string Description => "Operation Control Register Tests"; [HarpTest(Description = "Validates that OP_MODE bits can be round-tripped between Standby (0) and Active (1).")] @@ -32,15 +32,7 @@ public async Task OpModeRoundTrip(string portName) } finally { - // Always restore original state - try - { - await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, original)); - } - catch - { - // Ignore errors during restoration - } + await RestoreOperationControlAsync(device, original); } } } @@ -76,24 +68,21 @@ public async Task VisualEnWritable(string portName) public async Task HeartbeatEnEmitsEvents(string portName) { byte originalOpCtrl = 0; - ushort whoAmI = 0; try { using (var device = new AsyncDevice(portName)) { originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); - whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } await Task.Delay(500); // The previous one needs some time to disconnect - var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; var messages = await RegisterHelpers.WriteToTransportAsync( portName, - new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0xE5) }, + new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x05) }, TimeSpan.FromSeconds(2.0)); - bool received = messages.Any(m => m.Address == 18 && m.MessageType == MessageType.Event); + bool received = messages.Any(m => m.Address == R_HEARTBEAT.Address && m.MessageType == MessageType.Event); return new AssertionResult( received, @@ -107,11 +96,7 @@ public async Task HeartbeatEnEmitsEvents(string portName) } finally { - await Task.Delay(200); // Wait for port to be released before reopening - using (var device = new AsyncDevice(portName)) - { - await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, originalOpCtrl)); - } + await RestoreOperationControlAsync(portName, originalOpCtrl); } } @@ -119,26 +104,23 @@ public async Task HeartbeatEnEmitsEvents(string portName) public async Task HeartbeatEnPrecedenceOverAliveEn(string portName) { byte originalOpCtrl = 0; - ushort whoAmI = 0; try { using (var device = new AsyncDevice(portName)) { originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); - whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } await Task.Delay(500); - var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; // Set both ALIVE_EN (bit 7) and HEARTBEAT_EN (bit 2) with Active mode (bit 0) var messages = await RegisterHelpers.WriteToTransportAsync( portName, new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x85) }, TimeSpan.FromSeconds(2.0)); - bool receivedHeartbeat = messages.Any(m => m.Address == 18 && m.MessageType == MessageType.Event); - bool receivedTimestamp = messages.Any(m => m.Address == 8 && m.MessageType == MessageType.Event); + bool receivedHeartbeat = messages.Any(m => m.Address == R_HEARTBEAT.Address && m.MessageType == MessageType.Event); + bool receivedTimestamp = messages.Any(m => m.Address == TimestampSeconds.Address && m.MessageType == MessageType.Event); if (!receivedHeartbeat) return new AssertionResult(false, "HeartbeatEnPrecedenceOverAliveEn: no R_HEARTBEAT event received within 2s (expected HEARTBEAT_EN to take precedence)."); @@ -153,11 +135,7 @@ public async Task HeartbeatEnPrecedenceOverAliveEn(string portName) } finally { - await Task.Delay(200); - using (var device = new AsyncDevice(portName)) - { - await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, originalOpCtrl)); - } + await RestoreOperationControlAsync(portName, originalOpCtrl); } } @@ -165,25 +143,22 @@ public async Task HeartbeatEnPrecedenceOverAliveEn(string portName) public async Task AliveEnEmitsTimestampEvents(string portName) { byte originalOpCtrl = 0; - ushort whoAmI = 0; try { using (var device = new AsyncDevice(portName)) { originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); - whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } await Task.Delay(500); - var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; // Set only ALIVE_EN (bit 7) with Active mode (bit 0); HEARTBEAT_EN (bit 2) is cleared var messages = await RegisterHelpers.WriteToTransportAsync( portName, new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x81) }, TimeSpan.FromSeconds(2.0)); - bool receivedTimestamp = messages.Any(m => m.Address == 8 && m.MessageType == MessageType.Event); + bool receivedTimestamp = messages.Any(m => m.Address == TimestampSeconds.Address && m.MessageType == MessageType.Event); if (!receivedTimestamp) return new Result(false, Status.Skipped, "AliveEnEmitsTimestampEvents: ALIVE_EN is deprecated and R_TIMESTAMP_SECOND events were not emitted."); @@ -196,11 +171,7 @@ public async Task AliveEnEmitsTimestampEvents(string portName) } finally { - await Task.Delay(200); - using (var device = new AsyncDevice(portName)) - { - await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, originalOpCtrl)); - } + await RestoreOperationControlAsync(portName, originalOpCtrl); } } @@ -208,7 +179,6 @@ public async Task AliveEnEmitsTimestampEvents(string portName) public async Task RegisterDump(string portName) { byte originalOpCtrl = 0; - ushort whoAmI = 0; try { @@ -216,10 +186,9 @@ public async Task RegisterDump(string portName) using (var device = new AsyncDevice(portName)) { originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); - whoAmI = await device.ReadUInt16Async(WhoAmI.Address); } + await Task.Delay(500); // The previous one needs some time to disconnect - var harpDevice = new Bonsai.Harp.Device(whoAmI) { PortName = portName }; var messages = await RegisterHelpers.WriteToTransportAsync( portName, new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, (byte)(originalOpCtrl | 0x08)) }, @@ -232,13 +201,13 @@ public async Task RegisterDump(string portName) } var coreReads = messages .Select((m, i) => (msg: m, idx: i)) - .Where(x => x.msg.Address <= 32 && x.msg.MessageType == MessageType.Read) + .Where(x => x.msg.Address < 32 && x.msg.MessageType == MessageType.Read) .ToList(); var uniqueCoreAddresses = coreReads.Select(x => x.msg.Address).Distinct().ToHashSet(); - var missing = Enumerable.Range(0, 18).Where(a => !uniqueCoreAddresses.Contains(a)).ToList(); + var missing = Enumerable.Range(0, 20).Where(a => !uniqueCoreAddresses.Contains(a)).ToList(); if (missing.Count > 0) return new AssertionResult(false, - $"Missing Read replies for {missing.Count} core address(es): {string.Join(", ", missing.Select(a => $"0x{a:X2}"))}."); + $"Missing Read replies for {missing.Count} core address(es): {string.Join(", ", missing)}."); return new AssertionResult(true, "All core register reads received after OpCtrl write."); } @@ -248,12 +217,31 @@ public async Task RegisterDump(string portName) } finally { - await Task.Delay(200); - // Ensure we restore original state even though DUMP is transient - using (var device = new AsyncDevice(portName)) - { - await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, originalOpCtrl)); - } + await RestoreOperationControlAsync(portName, originalOpCtrl); + } + } + + private static async Task RestoreOperationControlAsync(AsyncDevice device, byte value) + { + try + { + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, value)); + } + catch + { + } + } + + private static async Task RestoreOperationControlAsync(string portName, byte value) + { + await Task.Delay(PortReleaseDelayMilliseconds); + try + { + using var device = new AsyncDevice(portName); + await RestoreOperationControlAsync(device, value); + } + catch + { } } @@ -285,13 +273,7 @@ private static async Task TestOptionalBitAsync(AsyncDevice device, stri } finally { - try - { - await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, original)); - } - catch - { - } + await RestoreOperationControlAsync(device, original); } } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs index b615f2a..577dd85 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs @@ -39,7 +39,7 @@ public async Task IsNotWritable(string portName) { using (var device = new AsyncDevice(portName)) { - var req = HarpMessage.FromByte(Address, MessageType.Write, 0x00); + var req = HarpMessage.FromByte(Address, MessageType.Write, new byte[ExpectedLength]); var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( rejected, diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs index 9738f47..286bb2a 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs @@ -18,7 +18,7 @@ public async Task IsWritable(string portName) HarpMessage response = await device.CommandAsync(TimestampSeconds.FromPayload(MessageType.Read, default)); double readSeconds = response.GetTimestamp(); return new AssertionResult( - readSeconds - setSeconds < 1.0, + Math.Abs(readSeconds - setSeconds) < 1.0, (success) => success ? "TimestampSeconds register is writable and updates as expected." : $"TimestampSeconds register is not writable. Expected value: {setSeconds}, read value: {readSeconds}."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs index 7bfcb89..917ced9 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs @@ -37,7 +37,7 @@ public async Task IsNotWritable(string portName) { using (var device = new AsyncDevice(portName)) { - var req = HarpMessage.FromByte(Version.Address, MessageType.Write, 0x00); + var req = HarpMessage.FromByte(Version.Address, MessageType.Write, new byte[Version.RegisterLength]); var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); return new AssertionResult( rejected, diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index 7897118..4b88aad 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -32,16 +32,21 @@ public VerifyCommand() Option ppsEventOption = new("--pps-event") { - Description = "Address of the register on the tested device (--port) that reports the incoming PPS pulse from the reference clock device. Enables the PPS alignment test.", + Description = "Address of the register on the tested device (--port) that reports the incoming PPS pulse from the reference clock device. Enables the PPS alignment test, which also requires --clock-port.", Required = false, }; Option clockSamplesOption = new("--clock-samples") { - Description = "Number of PPS event pairs to collect for the PPS alignment test. Default: 5.", + Description = "Number of PPS event pairs to collect for the PPS alignment test.", Required = false, }; clockSamplesOption.DefaultValueFactory = _ => 5; + clockSamplesOption.Validators.Add(result => + { + if (result.GetValueOrDefault() < 1) + result.AddError("The number of clock samples must be greater than zero."); + }); Option deviceYmlOption = new("--device-yml") { @@ -176,7 +181,7 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve { AnsiConsole.Markup("Generating HTML report..."); string html = await HtmlReportGenerator.GenerateAsync(report); - string fileName = reportFile?.FullName ?? $"TestReport_{DateTime.Now:yyyyMMdd_HHmmss}.html"; + string fileName = reportFile.FullName; await File.WriteAllTextAsync(fileName, html, cancellationToken); AnsiConsole.MarkupLine($"[green]Done![/] Report generated: [link]{fileName}[/]"); } From 2158f7f69c64a357c3c887ad861137d04a92038f Mon Sep 17 00:00:00 2001 From: glopesdev Date: Thu, 3 Sep 2026 21:42:37 +0100 Subject: [PATCH 24/41] Share one device connection across verify tests Tests now receive a shared connection instead of a port name, so a run opens the device once rather than once per test. A sweep goes from 49 port openings to one, or two when a reference clock device is configured, and completes in about half the time. Repeated runs of the same build now produce the same result for every test that does not depend on host timing. A denied open is retried against a ten second deadline rather than failing immediately. --- .../GenerateRegisterMetadataCommand.cs | 2 - src/Harp.Toolkit/Program.cs | 1 + .../Verify/GeneratedInterfaceCompiler.cs | 1 - src/Harp.Toolkit/Verify/HarpTestAttribute.cs | 2 +- .../Verify/HtmlReportGenerator.cs | 2 +- src/Harp.Toolkit/Verify/Report.cs | 2 +- src/Harp.Toolkit/Verify/ReportTemplate.cshtml | 4 +- src/Harp.Toolkit/Verify/Result.cs | 2 +- src/Harp.Toolkit/Verify/Runner.cs | 6 +- src/Harp.Toolkit/Verify/Suite.cs | 10 +- .../Verify/Suites/ClockTestSuite.cs | 18 +- .../CoreRegisters/R_ASSEMBLY_VERSION.cs | 21 +- .../Suites/CoreRegisters/R_CLOCK_CONFIG.cs | 36 ++- .../Suites/CoreRegisters/R_CORE_VERSION_H.cs | 19 +- .../Suites/CoreRegisters/R_CORE_VERSION_L.cs | 19 +- .../Suites/CoreRegisters/R_DEVICE_NAME.cs | 26 +- .../Suites/CoreRegisters/R_FW_VERSION_H.cs | 19 +- .../Suites/CoreRegisters/R_FW_VERSION_L.cs | 19 +- .../Suites/CoreRegisters/R_HEARTBEAT.cs | 26 +- .../Suites/CoreRegisters/R_HW_VERSION_H.cs | 19 +- .../Suites/CoreRegisters/R_HW_VERSION_L.cs | 19 +- .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 139 ++++------- .../Suites/CoreRegisters/R_RESET_DEV.cs | 7 +- .../Suites/CoreRegisters/R_SERIAL_NUMBER.cs | 29 +-- .../Verify/Suites/CoreRegisters/R_TAG.cs | 45 ++-- .../Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs | 55 ++--- .../CoreRegisters/R_TIMESTAMP_OFFSET.cs | 36 ++- .../CoreRegisters/R_TIMESTAMP_SECOND.cs | 88 +++---- .../Verify/Suites/CoreRegisters/R_UID.cs | 38 ++- .../Verify/Suites/CoreRegisters/R_VERSION.cs | 88 +++---- .../Verify/Suites/CoreRegisters/R_WHO_AM_I.cs | 34 ++- .../Suites/CoreRegisters/RegisterHelpers.cs | 45 +--- .../Verify/Suites/DeviceInterfaceSuite.cs | 11 +- .../Verify/Suites/RoundTripTestSuite.cs | 11 +- src/Harp.Toolkit/Verify/VerifyCommand.cs | 7 +- src/Harp.Toolkit/Verify/VerifyConnection.cs | 222 ++++++++++++++++++ 36 files changed, 570 insertions(+), 558 deletions(-) create mode 100644 src/Harp.Toolkit/Verify/VerifyConnection.cs diff --git a/src/Harp.Toolkit/Generate/GenerateRegisterMetadataCommand.cs b/src/Harp.Toolkit/Generate/GenerateRegisterMetadataCommand.cs index 0975834..07e734f 100644 --- a/src/Harp.Toolkit/Generate/GenerateRegisterMetadataCommand.cs +++ b/src/Harp.Toolkit/Generate/GenerateRegisterMetadataCommand.cs @@ -3,8 +3,6 @@ using Bonsai.Harp; using ExcelDataReader; using Harp.Generators; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; namespace Harp.Toolkit.Generate; diff --git a/src/Harp.Toolkit/Program.cs b/src/Harp.Toolkit/Program.cs index 87076d9..7fc160e 100644 --- a/src/Harp.Toolkit/Program.cs +++ b/src/Harp.Toolkit/Program.cs @@ -1,6 +1,7 @@ using System.CommandLine; using Bonsai.Harp; using Harp.Toolkit.Generate; +using Harp.Toolkit.Verify; namespace Harp.Toolkit; diff --git a/src/Harp.Toolkit/Verify/GeneratedInterfaceCompiler.cs b/src/Harp.Toolkit/Verify/GeneratedInterfaceCompiler.cs index cf23c1b..a2239b0 100644 --- a/src/Harp.Toolkit/Verify/GeneratedInterfaceCompiler.cs +++ b/src/Harp.Toolkit/Verify/GeneratedInterfaceCompiler.cs @@ -4,7 +4,6 @@ using Harp.Toolkit.Generate; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; -using Microsoft.CodeAnalysis.Emit; using Microsoft.Extensions.DependencyModel; namespace Harp.Toolkit.Verify; diff --git a/src/Harp.Toolkit/Verify/HarpTestAttribute.cs b/src/Harp.Toolkit/Verify/HarpTestAttribute.cs index f0bf7cb..701d5b6 100644 --- a/src/Harp.Toolkit/Verify/HarpTestAttribute.cs +++ b/src/Harp.Toolkit/Verify/HarpTestAttribute.cs @@ -1,4 +1,4 @@ -namespace Harp.Toolkit; +namespace Harp.Toolkit.Verify; [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public class HarpTestAttribute : Attribute diff --git a/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs b/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs index 5686c34..6b72e72 100644 --- a/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs +++ b/src/Harp.Toolkit/Verify/HtmlReportGenerator.cs @@ -1,7 +1,7 @@ using System.Reflection; using RazorLight; -namespace Harp.Toolkit; +namespace Harp.Toolkit.Verify; public static class HtmlReportGenerator { diff --git a/src/Harp.Toolkit/Verify/Report.cs b/src/Harp.Toolkit/Verify/Report.cs index cc716f2..3e387d2 100644 --- a/src/Harp.Toolkit/Verify/Report.cs +++ b/src/Harp.Toolkit/Verify/Report.cs @@ -1,4 +1,4 @@ -namespace Harp.Toolkit; +namespace Harp.Toolkit.Verify; public class Report { diff --git a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml index 9cdfbac..0b9b66e 100644 --- a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml +++ b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml @@ -1,5 +1,5 @@ -@using Harp.Toolkit -@model Harp.Toolkit.Report +@using Harp.Toolkit.Verify +@model Harp.Toolkit.Verify.Report diff --git a/src/Harp.Toolkit/Verify/Result.cs b/src/Harp.Toolkit/Verify/Result.cs index 312f97a..a5cb605 100644 --- a/src/Harp.Toolkit/Verify/Result.cs +++ b/src/Harp.Toolkit/Verify/Result.cs @@ -1,5 +1,5 @@  -namespace Harp.Toolkit; +namespace Harp.Toolkit.Verify; public enum Status diff --git a/src/Harp.Toolkit/Verify/Runner.cs b/src/Harp.Toolkit/Verify/Runner.cs index f243661..4d9484f 100644 --- a/src/Harp.Toolkit/Verify/Runner.cs +++ b/src/Harp.Toolkit/Verify/Runner.cs @@ -1,6 +1,6 @@ using System.Runtime.CompilerServices; -namespace Harp.Toolkit; +namespace Harp.Toolkit.Verify; public class Runner { @@ -17,11 +17,11 @@ public IEnumerable CollectSuites() return suites.AsReadOnly(); } - public async IAsyncEnumerable<(Suite Suite, MethodResult Result)> RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default, Action? onTestStart = null) + public async IAsyncEnumerable<(Suite Suite, MethodResult Result)> RunAllAsync(VerifyConnection connection, [EnumeratorCancellation] CancellationToken cancellationToken = default, Action? onTestStart = null) { foreach (var suite in suites) { - await foreach (var result in suite.RunAllAsync(portName, cancellationToken, (testName, testDesc) => onTestStart?.Invoke(suite, testName, testDesc))) + await foreach (var result in suite.RunAllAsync(connection, cancellationToken, (testName, testDesc) => onTestStart?.Invoke(suite, testName, testDesc))) { yield return (suite, result); } diff --git a/src/Harp.Toolkit/Verify/Suite.cs b/src/Harp.Toolkit/Verify/Suite.cs index 1278b68..ef41f58 100644 --- a/src/Harp.Toolkit/Verify/Suite.cs +++ b/src/Harp.Toolkit/Verify/Suite.cs @@ -2,7 +2,7 @@ using System.Runtime.CompilerServices; using Bonsai.Harp; -namespace Harp.Toolkit; +namespace Harp.Toolkit.Verify; public abstract class Suite @@ -26,7 +26,7 @@ public abstract class Suite .Where(x => x.Attribute != null); } - public async IAsyncEnumerable RunAllAsync(string portName, [EnumeratorCancellation] CancellationToken cancellationToken = default, Action? onTestStart = null) + public async IAsyncEnumerable RunAllAsync(VerifyConnection connection, [EnumeratorCancellation] CancellationToken cancellationToken = default, Action? onTestStart = null) { foreach (var (method, attr) in CollectTests()) { @@ -38,7 +38,7 @@ public async IAsyncEnumerable RunAllAsync(string portName, [Enumer IResult testResult; try { - object? resultObj = method.Invoke(this, new object[] { portName }); + object? resultObj = method.Invoke(this, new object[] { connection }); if (resultObj is Task task) { testResult = await task; @@ -73,7 +73,7 @@ public async IAsyncEnumerable RunAllAsync(string portName, [Enumer IResult testResult; try { - testResult = await test.Run(portName, cancellationToken); + testResult = await test.Run(connection, cancellationToken); } catch (Exception ex) { @@ -93,7 +93,7 @@ public async IAsyncEnumerable RunAllAsync(string portName, [Enumer /// A test whose name and behavior is determined at runtime rather than declared /// with on a fixed method. /// -public record DynamicTest(string Name, string Description, Func> Run); +public record DynamicTest(string Name, string Description, Func> Run); public class SuiteResult { diff --git a/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs b/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs index 5ea922e..dd63e30 100644 --- a/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/ClockTestSuite.cs @@ -1,6 +1,5 @@  using Bonsai.Harp; -using Harp.Toolkit.Verify; namespace Harp.Toolkit.Verify.Suites; @@ -16,7 +15,7 @@ public ClockTestSuite(ClockTestOptions? options) public override string Description => "Tests clock alignment and PPS synchronization accuracy against a reference clock device."; [HarpTest(Description = "Sends 100 simultaneous WhoAmI reads to both devices and compares embedded timestamps to measure clock offset.")] - public async Task SimultaneousWhoAmI(string portName) + public async Task SimultaneousWhoAmI(VerifyConnection device) { if (options is null) return new Result(false, Status.Skipped, "No clock port provided (--clock-port)."); @@ -25,12 +24,11 @@ public async Task SimultaneousWhoAmI(string portName) double[] deltas = new double[n]; var probe = WhoAmI.FromPayload(MessageType.Read, default); - using var testedDevice = new AsyncDevice(portName); - using var clockDevice = new AsyncDevice(options.ClockPort); + using var clockDevice = await VerifyConnection.OpenAsync(options.ClockPort); for (int i = 0; i < n; i++) { - var results = await Task.WhenAll(testedDevice.CommandAsync(probe), clockDevice.CommandAsync(probe)); + var results = await Task.WhenAll(device.CommandAsync(probe), clockDevice.CommandAsync(probe)); deltas[i] = results[0].GetTimestamp() - results[1].GetTimestamp(); await Task.Delay(Random.Shared.Next(20, 70)); } @@ -44,7 +42,7 @@ public async Task SimultaneousWhoAmI(string portName) } [HarpTest(Description = "Subscribes to PPS events on both devices and compares timestamps to measure hardware clock synchronization accuracy.")] - public async Task PpsEventAlignment(string portName) + public async Task PpsEventAlignment(VerifyConnection device) { if (options is null) return new Result(false, Status.Skipped, "No clock port provided (--clock-port)."); @@ -58,13 +56,13 @@ public async Task PpsEventAlignment(string portName) const byte clockDeviceOpCtrl = 0x81; var listenDuration = TimeSpan.FromSeconds(options.ClockSamples + 5); + using var clockDevice = await VerifyConnection.OpenAsync(options.ClockPort); + var allMessages = await Task.WhenAll( - RegisterHelpers.WriteToTransportAsync( - options.ClockPort, + clockDevice.WriteAndCollectAsync( [HarpMessage.FromByte(OperationControl.Address, MessageType.Write, clockDeviceOpCtrl)], listenDuration), - RegisterHelpers.WriteToTransportAsync( - portName, + device.WriteAndCollectAsync( [HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x01)], listenDuration)); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs index df28c24..205ebae 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_ASSEMBLY_VERSION.cs @@ -1,22 +1,17 @@ - -using Bonsai.Harp; -namespace Harp.Toolkit.Verify.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_ASSEMBLY_VERSION : Suite { public override string Description => "AssemblyVersion Register Tests"; [HarpTest(Description = "Validates the deprecated register AssemblyVersion returns 0x00.")] - public async Task AssertReturnsZero(string portName) + public async Task AssertReturnsZero(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var value = await device.ReadAssemblyVersionAsync(); - return new AssertionResult( - value == 0x00, - x => x ? - "AssemblyVersion register correctly returned 0x00." : - $"AssemblyVersion register returned a non-zero value (0x{value:X2})."); - } + var value = await device.ReadAssemblyVersionAsync(); + return new AssertionResult( + value == 0x00, + x => x ? + "AssemblyVersion register correctly returned 0x00." : + $"AssemblyVersion register returned a non-zero value (0x{value:X2})."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs index c411c2a..ccbd64d 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CLOCK_CONFIG.cs @@ -8,31 +8,25 @@ internal class R_CLOCK_CONFIG : Suite public override string Description => "Clock Configuration Register Tests"; [HarpTest(Description = "Validates that ClockConfig register is readable.")] - public async Task IsReadable(string portName) + public async Task IsReadable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), ClockConfiguration.Address, "ClockConfig"); - } + return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), ClockConfiguration.Address, "ClockConfig"); } [HarpTest(Description = "Reports clock synchronization capability: REP_ABLE (bit 3) and GEN_ABLE (bit 4).")] - public async Task ReportSyncCapability(string portName) + public async Task ReportSyncCapability(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var value = await device.ReadByteAsync(ClockConfiguration.Address); - bool repAble = (value & (1 << 3)) != 0; - bool genAble = (value & (1 << 4)) != 0; - StringBuilder sb = new StringBuilder("ClockConfig sync capability:"); - sb.Append("\n"); - sb.Append(repAble ? "Device can repeat clock signal" : "Device cannot repeat clock signal"); - sb.Append("\n"); - sb.Append(genAble ? "Device can generate clock signal" : "Device cannot generate clock signal"); - sb.Append("\n"); - return new AssertionResult( - true, - sb.ToString()); - } + var value = await device.ReadByteAsync(ClockConfiguration.Address); + bool repAble = (value & (1 << 3)) != 0; + bool genAble = (value & (1 << 4)) != 0; + StringBuilder sb = new StringBuilder("ClockConfig sync capability:"); + sb.Append("\n"); + sb.Append(repAble ? "Device can repeat clock signal" : "Device cannot repeat clock signal"); + sb.Append("\n"); + sb.Append(genAble ? "Device can generate clock signal" : "Device cannot generate clock signal"); + sb.Append("\n"); + return new AssertionResult( + true, + sb.ToString()); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs index f706ce0..444a527 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs @@ -7,17 +7,14 @@ internal class R_CORE_VERSION_H : Suite public override string Description => "Core Version High Register Tests"; [HarpTest(Description = "Validates that CoreVersionHigh matches byte 0 of R_VERSION.")] - public async Task AssertConsistentWithVersion(string portName) + public async Task AssertConsistentWithVersion(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var versionArray = await device.ReadByteArrayAsync(Version.Address); - var registerValue = await device.ReadByteAsync(CoreVersionHigh.Address); - return new AssertionResult( - registerValue == versionArray[0], - x => x - ? $"CoreVersionHigh (0x{registerValue:X2}) matches R_VERSION byte 0." - : $"CoreVersionHigh (0x{registerValue:X2}) does not match R_VERSION byte 0 (0x{versionArray[0]:X2})."); - } + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(CoreVersionHigh.Address); + return new AssertionResult( + registerValue == versionArray[0], + x => x + ? $"CoreVersionHigh (0x{registerValue:X2}) matches R_VERSION byte 0." + : $"CoreVersionHigh (0x{registerValue:X2}) does not match R_VERSION byte 0 (0x{versionArray[0]:X2})."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs index a7e526e..635cbb1 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs @@ -7,17 +7,14 @@ internal class R_CORE_VERSION_L : Suite public override string Description => "Core Version Low Register Tests"; [HarpTest(Description = "Validates that CoreVersionLow matches byte 1 of R_VERSION.")] - public async Task AssertConsistentWithVersion(string portName) + public async Task AssertConsistentWithVersion(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var versionArray = await device.ReadByteArrayAsync(Version.Address); - var registerValue = await device.ReadByteAsync(CoreVersionLow.Address); - return new AssertionResult( - registerValue == versionArray[1], - x => x - ? $"CoreVersionLow (0x{registerValue:X2}) matches R_VERSION byte 1." - : $"CoreVersionLow (0x{registerValue:X2}) does not match R_VERSION byte 1 (0x{versionArray[1]:X2})."); - } + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(CoreVersionLow.Address); + return new AssertionResult( + registerValue == versionArray[1], + x => x + ? $"CoreVersionLow (0x{registerValue:X2}) matches R_VERSION byte 1." + : $"CoreVersionLow (0x{registerValue:X2}) does not match R_VERSION byte 1 (0x{versionArray[1]:X2})."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs index cf63fe7..851942c 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_DEVICE_NAME.cs @@ -7,28 +7,22 @@ internal class R_DEVICE_NAME : Suite public override string Description => "Device Name Register Tests"; [HarpTest(Description = "Validates that DeviceName register is readable.")] - public async Task IsReadable(string portName) + public async Task IsReadable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) + try { - try - { - await device.ReadByteArrayAsync(DeviceName.Address); - return new AssertionResult(true, "DeviceName is readable."); - } - catch (Exception ex) - { - return new ErrorResult(ex); - } + await device.ReadByteArrayAsync(DeviceName.Address); + return new AssertionResult(true, "DeviceName is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); } } [HarpTest(Description = "Validates that DeviceName register has exactly 25 bytes.")] - public async Task AssertLength(string portName) + public async Task AssertLength(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - return await RegisterHelpers.AssertReadableArrayAsync(device, DeviceName.Address, DeviceName.RegisterLength, "DeviceName"); - } + return await RegisterHelpers.AssertReadableArrayAsync(device, DeviceName.Address, DeviceName.RegisterLength, "DeviceName"); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs index 616486e..6f344d4 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs @@ -7,17 +7,14 @@ internal class R_FW_VERSION_H : Suite public override string Description => "Firmware Version High Register Tests"; [HarpTest(Description = "Validates that FwVersionHigh matches byte 3 of R_VERSION.")] - public async Task AssertConsistentWithVersion(string portName) + public async Task AssertConsistentWithVersion(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var versionArray = await device.ReadByteArrayAsync(Version.Address); - var registerValue = await device.ReadByteAsync(FirmwareVersionHigh.Address); - return new AssertionResult( - registerValue == versionArray[3], - x => x - ? $"FwVersionHigh (0x{registerValue:X2}) matches R_VERSION byte 3." - : $"FwVersionHigh (0x{registerValue:X2}) does not match R_VERSION byte 3 (0x{versionArray[3]:X2})."); - } + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(FirmwareVersionHigh.Address); + return new AssertionResult( + registerValue == versionArray[3], + x => x + ? $"FwVersionHigh (0x{registerValue:X2}) matches R_VERSION byte 3." + : $"FwVersionHigh (0x{registerValue:X2}) does not match R_VERSION byte 3 (0x{versionArray[3]:X2})."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs index b2d95a1..7b61f5f 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs @@ -7,17 +7,14 @@ internal class R_FW_VERSION_L : Suite public override string Description => "Firmware Version Low Register Tests"; [HarpTest(Description = "Validates that FwVersionLow matches byte 4 of R_VERSION.")] - public async Task AssertConsistentWithVersion(string portName) + public async Task AssertConsistentWithVersion(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var versionArray = await device.ReadByteArrayAsync(Version.Address); - var registerValue = await device.ReadByteAsync(FirmwareVersionLow.Address); - return new AssertionResult( - registerValue == versionArray[4], - x => x - ? $"FwVersionLow (0x{registerValue:X2}) matches R_VERSION byte 4." - : $"FwVersionLow (0x{registerValue:X2}) does not match R_VERSION byte 4 (0x{versionArray[4]:X2})."); - } + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(FirmwareVersionLow.Address); + return new AssertionResult( + registerValue == versionArray[4], + x => x + ? $"FwVersionLow (0x{registerValue:X2}) matches R_VERSION byte 4." + : $"FwVersionLow (0x{registerValue:X2}) does not match R_VERSION byte 4 (0x{versionArray[4]:X2})."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs index 7fa7f47..788a772 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs @@ -8,26 +8,20 @@ internal class R_HEARTBEAT : Suite public override string Description => "Heartbeat Register Tests"; [HarpTest(Description = "Validates that Heartbeat register is readable.")] - public async Task IsReadable(string portName) + public async Task IsReadable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - return await RegisterHelpers.AssertReadableAsync(a => device.ReadUInt16Async(a), Address, "Heartbeat"); - } + return await RegisterHelpers.AssertReadableAsync(a => device.ReadUInt16Async(a), Address, "Heartbeat"); } [HarpTest(Description = "Validates that Heartbeat register is NOT writable.")] - public async Task IsNotWritable(string portName) + public async Task IsNotWritable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var req = HarpMessage.FromUInt16(Address, MessageType.Write, 0); - var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); - return new AssertionResult( - rejected, - x => x - ? "Heartbeat register correctly rejected write." - : "Heartbeat register should NOT be writable."); - } + var req = HarpMessage.FromUInt16(Address, MessageType.Write, 0); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x + ? "Heartbeat register correctly rejected write." + : "Heartbeat register should NOT be writable."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs index efe1009..c5a2581 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs @@ -7,17 +7,14 @@ internal class R_HW_VERSION_H : Suite public override string Description => "Hardware Version High Register Tests"; [HarpTest(Description = "Validates that HwVersionHigh matches byte 6 of R_VERSION.")] - public async Task AssertConsistentWithVersion(string portName) + public async Task AssertConsistentWithVersion(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var versionArray = await device.ReadByteArrayAsync(Version.Address); - var registerValue = await device.ReadByteAsync(HardwareVersionHigh.Address); - return new AssertionResult( - registerValue == versionArray[6], - x => x - ? $"HwVersionHigh (0x{registerValue:X2}) matches R_VERSION byte 6." - : $"HwVersionHigh (0x{registerValue:X2}) does not match R_VERSION byte 6 (0x{versionArray[6]:X2})."); - } + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(HardwareVersionHigh.Address); + return new AssertionResult( + registerValue == versionArray[6], + x => x + ? $"HwVersionHigh (0x{registerValue:X2}) matches R_VERSION byte 6." + : $"HwVersionHigh (0x{registerValue:X2}) does not match R_VERSION byte 6 (0x{versionArray[6]:X2})."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs index 9781a52..85e9078 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs @@ -7,17 +7,14 @@ internal class R_HW_VERSION_L : Suite public override string Description => "Hardware Version Low Register Tests"; [HarpTest(Description = "Validates that HwVersionLow matches byte 7 of R_VERSION.")] - public async Task AssertConsistentWithVersion(string portName) + public async Task AssertConsistentWithVersion(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var versionArray = await device.ReadByteArrayAsync(Version.Address); - var registerValue = await device.ReadByteAsync(HardwareVersionLow.Address); - return new AssertionResult( - registerValue == versionArray[7], - x => x - ? $"HwVersionLow (0x{registerValue:X2}) matches R_VERSION byte 7." - : $"HwVersionLow (0x{registerValue:X2}) does not match R_VERSION byte 7 (0x{versionArray[7]:X2})."); - } + var versionArray = await device.ReadByteArrayAsync(Version.Address); + var registerValue = await device.ReadByteAsync(HardwareVersionLow.Address); + return new AssertionResult( + registerValue == versionArray[7], + x => x + ? $"HwVersionLow (0x{registerValue:X2}) matches R_VERSION byte 7." + : $"HwVersionLow (0x{registerValue:X2}) does not match R_VERSION byte 7 (0x{versionArray[7]:X2})."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs index a04cdba..22de604 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -1,84 +1,64 @@ -using Bonsai.Harp; +using Bonsai.Harp; namespace Harp.Toolkit.Verify.Suites; internal class R_OPERATION_CTRL : Suite { - private const int PortReleaseDelayMilliseconds = 200; - public override string Description => "Operation Control Register Tests"; [HarpTest(Description = "Validates that OP_MODE bits can be round-tripped between Standby (0) and Active (1).")] - public async Task OpModeRoundTrip(string portName) + public async Task OpModeRoundTrip(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var original = await device.ReadByteAsync(OperationControl.Address); - byte currentMode = (byte)(original & 0x03); - byte newMode = currentMode == 0x01 ? (byte)0x00 : (byte)0x01; - byte newValue = (byte)((original & ~0x03) | newMode); + var original = await device.ReadByteAsync(OperationControl.Address); + byte currentMode = (byte)(original & 0x03); + byte newMode = currentMode == 0x01 ? (byte)0x00 : (byte)0x01; + byte newValue = (byte)((original & ~0x03) | newMode); - try - { - await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, newValue)); - var readBack = await device.ReadByteAsync(OperationControl.Address); - byte readMode = (byte)(readBack & 0x03); + try + { + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, newValue)); + var readBack = await device.ReadByteAsync(OperationControl.Address); + byte readMode = (byte)(readBack & 0x03); - return new AssertionResult( - readMode == newMode, - x => x - ? $"OpModeRoundTrip: OP_MODE correctly round-tripped to {newMode}." - : $"OpModeRoundTrip: wrote OP_MODE={newMode}, read back OP_MODE={readMode}."); - } - finally - { - await RestoreOperationControlAsync(device, original); - } + return new AssertionResult( + readMode == newMode, + x => x + ? $"OpModeRoundTrip: OP_MODE correctly round-tripped to {newMode}." + : $"OpModeRoundTrip: wrote OP_MODE={newMode}, read back OP_MODE={readMode}."); + } + finally + { + await RestoreOperationControlAsync(device, original); } } [HarpTest(Description = "Validates that ALIVE_EN (deprecated, bit 7) can be toggled, or reports as unsupported.")] - public async Task AliveEnWritable(string portName) + public async Task AliveEnWritable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - return await TestOptionalBitAsync(device, "AliveEn", 0x80); - } + return await TestOptionalBitAsync(device, "AliveEn", 0x80); } [HarpTest(Description = "Validates that OPLED_EN (optional, bit 6) can be toggled, or reports as unsupported.")] - public async Task OpLedEnWritable(string portName) + public async Task OpLedEnWritable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - return await TestOptionalBitAsync(device, "OpLedEn", 0x40); - } + return await TestOptionalBitAsync(device, "OpLedEn", 0x40); } [HarpTest(Description = "Validates that VISUAL_EN (optional, bit 5) can be toggled, or reports as unsupported.")] - public async Task VisualEnWritable(string portName) + public async Task VisualEnWritable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - return await TestOptionalBitAsync(device, "VisualEn", 0x20); - } + return await TestOptionalBitAsync(device, "VisualEn", 0x20); } [HarpTest(Description = "Validates that enabling HEARTBEAT_EN causes the device to emit R_HEARTBEAT events.")] - public async Task HeartbeatEnEmitsEvents(string portName) + public async Task HeartbeatEnEmitsEvents(VerifyConnection device) { byte originalOpCtrl = 0; try { - using (var device = new AsyncDevice(portName)) - { - originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); - } - await Task.Delay(500); // The previous one needs some time to disconnect - - var messages = await RegisterHelpers.WriteToTransportAsync( - portName, + originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); + var messages = await device.WriteAndCollectAsync( new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x05) }, TimeSpan.FromSeconds(2.0)); @@ -96,26 +76,20 @@ public async Task HeartbeatEnEmitsEvents(string portName) } finally { - await RestoreOperationControlAsync(portName, originalOpCtrl); + await RestoreOperationControlAsync(device, originalOpCtrl); } } [HarpTest(Description = "Validates that HEARTBEAT_EN (bit 2) takes precedence over ALIVE_EN (bit 7): when both are set, R_HEARTBEAT events are emitted and R_TIMESTAMP_SECOND events are not.")] - public async Task HeartbeatEnPrecedenceOverAliveEn(string portName) + public async Task HeartbeatEnPrecedenceOverAliveEn(VerifyConnection device) { byte originalOpCtrl = 0; try { - using (var device = new AsyncDevice(portName)) - { - originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); - } - await Task.Delay(500); - + originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); // Set both ALIVE_EN (bit 7) and HEARTBEAT_EN (bit 2) with Active mode (bit 0) - var messages = await RegisterHelpers.WriteToTransportAsync( - portName, + var messages = await device.WriteAndCollectAsync( new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x85) }, TimeSpan.FromSeconds(2.0)); @@ -135,26 +109,20 @@ public async Task HeartbeatEnPrecedenceOverAliveEn(string portName) } finally { - await RestoreOperationControlAsync(portName, originalOpCtrl); + await RestoreOperationControlAsync(device, originalOpCtrl); } } [HarpTest(Description = "Validates that ALIVE_EN (deprecated, bit 7) causes R_TIMESTAMP_SECOND events to be emitted when HEARTBEAT_EN is not set.")] - public async Task AliveEnEmitsTimestampEvents(string portName) + public async Task AliveEnEmitsTimestampEvents(VerifyConnection device) { byte originalOpCtrl = 0; try { - using (var device = new AsyncDevice(portName)) - { - originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); - } - await Task.Delay(500); - + originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); // Set only ALIVE_EN (bit 7) with Active mode (bit 0); HEARTBEAT_EN (bit 2) is cleared - var messages = await RegisterHelpers.WriteToTransportAsync( - portName, + var messages = await device.WriteAndCollectAsync( new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, 0x81) }, TimeSpan.FromSeconds(2.0)); @@ -171,26 +139,20 @@ public async Task AliveEnEmitsTimestampEvents(string portName) } finally { - await RestoreOperationControlAsync(portName, originalOpCtrl); + await RestoreOperationControlAsync(device, originalOpCtrl); } } [HarpTest(Description = "Validates that the DUMP bit triggers a burst of all core register reads after an OpCtrl write.")] - public async Task RegisterDump(string portName) + public async Task RegisterDump(VerifyConnection device) { byte originalOpCtrl = 0; try { // Read original state before modifying - using (var device = new AsyncDevice(portName)) - { - originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); - } - await Task.Delay(500); // The previous one needs some time to disconnect - - var messages = await RegisterHelpers.WriteToTransportAsync( - portName, + originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); + var messages = await device.WriteAndCollectAsync( new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, (byte)(originalOpCtrl | 0x08)) }, TimeSpan.FromSeconds(1)); @@ -217,11 +179,11 @@ public async Task RegisterDump(string portName) } finally { - await RestoreOperationControlAsync(portName, originalOpCtrl); + await RestoreOperationControlAsync(device, originalOpCtrl); } } - private static async Task RestoreOperationControlAsync(AsyncDevice device, byte value) + private static async Task RestoreOperationControlAsync(VerifyConnection device, byte value) { try { @@ -232,20 +194,7 @@ private static async Task RestoreOperationControlAsync(AsyncDevice device, byte } } - private static async Task RestoreOperationControlAsync(string portName, byte value) - { - await Task.Delay(PortReleaseDelayMilliseconds); - try - { - using var device = new AsyncDevice(portName); - await RestoreOperationControlAsync(device, value); - } - catch - { - } - } - - private static async Task TestOptionalBitAsync(AsyncDevice device, string bitName, byte bitMask) + private static async Task TestOptionalBitAsync(VerifyConnection device, string bitName, byte bitMask) { var original = await device.ReadByteAsync(OperationControl.Address); byte toggled = (byte)(original ^ bitMask); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs index 405d11d..af08d03 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs @@ -7,11 +7,8 @@ internal class R_RESET_DEV : Suite public override string Description => "Reset Device Register Tests"; [HarpTest(Description = "Validates that ResetDev register is readable.")] - public async Task IsReadable(string portName) + public async Task IsReadable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), ResetDevice.Address, "ResetDev"); - } + return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), ResetDevice.Address, "ResetDev"); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs index a398840..a10843f 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs @@ -1,28 +1,23 @@ - -using Bonsai.Harp; -namespace Harp.Toolkit.Verify.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_SERIAL_NUMBER : Suite { public override string Description => "Serial Number Register Tests"; [HarpTest(Description = "Validates that SerialNumber matches the first two bytes of R_UID.")] - public async Task AssertConsistentWithUid(string portName) + public async Task AssertConsistentWithUid(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var uidValue = await device.ReadByteArrayAsync(R_UID.Address); - if (uidValue.Length < 2) - throw new ArgumentException($"Expected UID register contents to be at least 2 bytes. Got {uidValue.Length}."); - var twoFirstBytes = BitConverter.ToInt16(uidValue, 0); + var uidValue = await device.ReadByteArrayAsync(R_UID.Address); + if (uidValue.Length < 2) + throw new ArgumentException($"Expected UID register contents to be at least 2 bytes. Got {uidValue.Length}."); + var twoFirstBytes = BitConverter.ToInt16(uidValue, 0); - var serialNumberValue = await device.ReadSerialNumberAsync(); + var serialNumberValue = await device.ReadSerialNumberAsync(); - return new AssertionResult( - twoFirstBytes == serialNumberValue, - x => x ? - "SerialNumber register contents are consistent with UID register." : - $"SerialNumber register content (0x{serialNumberValue:X4}) does not match the first two bytes of UID register (0x{twoFirstBytes:X4})."); - } + return new AssertionResult( + twoFirstBytes == serialNumberValue, + x => x ? + "SerialNumber register contents are consistent with UID register." : + $"SerialNumber register content (0x{serialNumberValue:X4}) does not match the first two bytes of UID register (0x{twoFirstBytes:X4})."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs index 577dd85..b452e95 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs @@ -9,43 +9,34 @@ internal class R_TAG : Suite public override string Description => "Tag Register Tests"; [HarpTest(Description = "Validates that Tag register is readable.")] - public async Task IsReadable(string portName) + public async Task IsReadable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) + try { - try - { - await device.ReadByteArrayAsync(Address); - return new AssertionResult(true, "Tag is readable."); - } - catch (Exception ex) - { - return new ErrorResult(ex); - } + await device.ReadByteArrayAsync(Address); + return new AssertionResult(true, "Tag is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); } } [HarpTest(Description = "Validates that Tag register has exactly 8 bytes.")] - public async Task AssertLength(string portName) + public async Task AssertLength(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - return await RegisterHelpers.AssertReadableArrayAsync(device, Address, ExpectedLength, "Tag"); - } + return await RegisterHelpers.AssertReadableArrayAsync(device, Address, ExpectedLength, "Tag"); } [HarpTest(Description = "Validates that Tag register is NOT writable.")] - public async Task IsNotWritable(string portName) + public async Task IsNotWritable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var req = HarpMessage.FromByte(Address, MessageType.Write, new byte[ExpectedLength]); - var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); - return new AssertionResult( - rejected, - x => x - ? "Tag register correctly rejected write." - : "Tag register should NOT be writable."); - } + var req = HarpMessage.FromByte(Address, MessageType.Write, new byte[ExpectedLength]); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x + ? "Tag register correctly rejected write." + : "Tag register should NOT be writable."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs index beaacd6..42736b3 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_MICRO.cs @@ -7,48 +7,39 @@ internal class R_TIMESTAMP_MICRO : Suite public override string Description => "Timestamp Microseconds Register Tests"; [HarpTest(Description = "Validates that TimestampMicro register is readable.")] - public async Task IsReadable(string portName) + public async Task IsReadable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) + try { - try - { - await device.ReadUInt16Async(TimestampMicroseconds.Address); - return new AssertionResult(true, "TimestampMicro is readable."); - } - catch (Exception ex) - { - return new ErrorResult(ex); - } + await device.ReadUInt16Async(TimestampMicroseconds.Address); + return new AssertionResult(true, "TimestampMicro is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); } } [HarpTest(Description = "Validates that TimestampMicro register is NOT writable.")] - public async Task IsNotWritable(string portName) + public async Task IsNotWritable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var req = HarpMessage.FromUInt16(TimestampMicroseconds.Address, MessageType.Write, 0); - var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); - return new AssertionResult( - rejected, - x => x - ? "TimestampMicro register correctly rejected write." - : "TimestampMicro register should NOT be writable."); - } + var req = HarpMessage.FromUInt16(TimestampMicroseconds.Address, MessageType.Write, 0); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x + ? "TimestampMicro register correctly rejected write." + : "TimestampMicro register should NOT be writable."); } [HarpTest(Description = "Validates that TimestampMicro value is within bounds (0 to 31249).")] - public async Task ValueWithinBounds(string portName) + public async Task ValueWithinBounds(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var microValue = await device.ReadUInt16Async(TimestampMicroseconds.Address); - return new AssertionResult( - microValue < 31250, - x => x - ? $"TimestampMicro value ({microValue}) is within expected bounds (< 31250)." - : $"TimestampMicro value ({microValue}) exceeds expected maximum (31249)."); - } + var microValue = await device.ReadUInt16Async(TimestampMicroseconds.Address); + return new AssertionResult( + microValue < 31250, + x => x + ? $"TimestampMicro value ({microValue}) is within expected bounds (< 31250)." + : $"TimestampMicro value ({microValue}) exceeds expected maximum (31249)."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs index c73a198..4b273ee 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs @@ -8,31 +8,25 @@ internal class R_TIMESTAMP_OFFSET : Suite public override string Description => "Timestamp Offset Register Tests"; [HarpTest(Description = "Validates the deprecated register TimestampOffset returns 0x00.")] - public async Task AssertReturnsZero(string portName) + public async Task AssertReturnsZero(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var value = await device.ReadByteAsync(Address); - return new AssertionResult( - value == 0x00, - x => x ? - "TimestampOffset register correctly returned 0x00." : - $"TimestampOffset register returned a non-zero value (0x{value:X2})."); - } + var value = await device.ReadByteAsync(Address); + return new AssertionResult( + value == 0x00, + x => x ? + "TimestampOffset register correctly returned 0x00." : + $"TimestampOffset register returned a non-zero value (0x{value:X2})."); } [HarpTest(Description = "Validates the deprecated register TimestampOffset is NOT writable.")] - public async Task IsNotWritable(string portName) + public async Task IsNotWritable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var req = HarpMessage.FromByte(Address, MessageType.Write, 0x00); - var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); - return new AssertionResult( - rejected, - x => x ? - "Device correctly reported an error when trying to write to TimestampOffset register." : - "Timestamp Offset register is deprecated and MUST NOT allow writes."); - } + var req = HarpMessage.FromByte(Address, MessageType.Write, 0x00); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x ? + "Device correctly reported an error when trying to write to TimestampOffset register." : + "Timestamp Offset register is deprecated and MUST NOT allow writes."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs index 286bb2a..a326be9 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs @@ -8,74 +8,62 @@ internal class R_TIMESTAMP_SECOND : Suite public override string Description => "Timestamp Seconds Register Tests"; [HarpTest(Description = "Validates that the Timestamp Seconds register is writable.")] - public async Task IsWritable(string portName) + public async Task IsWritable(VerifyConnection device) { const uint setSeconds = 42; - using (var device = new AsyncDevice(portName)) - { - await device.WriteTimestampSecondsAsync(setSeconds); - await Task.Delay(1); - HarpMessage response = await device.CommandAsync(TimestampSeconds.FromPayload(MessageType.Read, default)); - double readSeconds = response.GetTimestamp(); - return new AssertionResult( - Math.Abs(readSeconds - setSeconds) < 1.0, - (success) => success ? "TimestampSeconds register is writable and updates as expected." : $"TimestampSeconds register is not writable. Expected value: {setSeconds}, read value: {readSeconds}."); - } + await device.WriteTimestampSecondsAsync(setSeconds); + await Task.Delay(1); + HarpMessage response = await device.CommandAsync(TimestampSeconds.FromPayload(MessageType.Read, default)); + double readSeconds = response.GetTimestamp(); + return new AssertionResult( + Math.Abs(readSeconds - setSeconds) < 1.0, + (success) => success ? "TimestampSeconds register is writable and updates as expected." : $"TimestampSeconds register is not writable. Expected value: {setSeconds}, read value: {readSeconds}."); } [HarpTest(Description = "Validates that TimestampSeconds register is readable.")] - public async Task IsReadable(string portName) + public async Task IsReadable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) + try + { + await device.ReadTimestampSecondsAsync(); + return new AssertionResult(true, "TimestampSeconds is readable."); + } + catch (Exception ex) { - try - { - await device.ReadTimestampSecondsAsync(); - return new AssertionResult(true, "TimestampSeconds is readable."); - } - catch (Exception ex) - { - return new ErrorResult(ex); - } + return new ErrorResult(ex); } } [HarpTest(Description = "Validates that TimestampSeconds register is monotonically non-decreasing.")] - public async Task IsMonotonic(string portName) + public async Task IsMonotonic(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var first = await device.ReadTimestampSecondsAsync(); - await Task.Delay(100); - var second = await device.ReadTimestampSecondsAsync(); - return new AssertionResult( - second >= first, - x => x - ? $"TimestampSeconds is monotonic: {first} -> {second}." - : $"TimestampSeconds decreased from {first} to {second}."); - } + var first = await device.ReadTimestampSecondsAsync(); + await Task.Delay(100); + var second = await device.ReadTimestampSecondsAsync(); + return new AssertionResult( + second >= first, + x => x + ? $"TimestampSeconds is monotonic: {first} -> {second}." + : $"TimestampSeconds decreased from {first} to {second}."); } [HarpTest(Description = "Validates that writing a past timestamp value takes effect and can be read back.")] - public async Task WritePastValueRoundTrip(string portName) + public async Task WritePastValueRoundTrip(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var sw = Stopwatch.StartNew(); - var current = await device.ReadTimestampSecondsAsync(); - var tPast = current >= 10 ? current - 10 : 0u; + var sw = Stopwatch.StartNew(); + var current = await device.ReadTimestampSecondsAsync(); + var tPast = current >= 10 ? current - 10 : 0u; - await device.WriteTimestampSecondsAsync(tPast); - await Task.Delay(50); + await device.WriteTimestampSecondsAsync(tPast); + await Task.Delay(50); - var readBack = await device.ReadTimestampSecondsAsync(); - bool withinBounds = Math.Abs((long)readBack - (long)tPast) <= 1; + var readBack = await device.ReadTimestampSecondsAsync(); + bool withinBounds = Math.Abs((long)readBack - (long)tPast) <= 1; - return new AssertionResult( - withinBounds, - x => x - ? $"WritePastValueRoundTrip: wrote {tPast}, read back {readBack} (within 1s tolerance)." - : $"WritePastValueRoundTrip: wrote {tPast}, read back {readBack} (difference {Math.Abs((long)readBack - (long)tPast)}s, expected <= 1)."); - } + return new AssertionResult( + withinBounds, + x => x + ? $"WritePastValueRoundTrip: wrote {tPast}, read back {readBack} (within 1s tolerance)." + : $"WritePastValueRoundTrip: wrote {tPast}, read back {readBack} (difference {Math.Abs((long)readBack - (long)tPast)}s, expected <= 1)."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs index 7fc7fb0..2b193bd 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs @@ -1,6 +1,4 @@ - -using Bonsai.Harp; -namespace Harp.Toolkit.Verify.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_UID : Suite { @@ -9,30 +7,24 @@ internal class R_UID : Suite public override string Description => "UID Register Tests"; [HarpTest(Description = "Validates that UID register has exactly 16 bytes.")] - public async Task AssertLength(string portName) + public async Task AssertLength(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var value = await device.ReadByteArrayAsync(Address); - return new AssertionResult( - value.Length == ExpectedLength, - x => x ? - $"Length is {ExpectedLength} as expected." : - $"Expected length of register to be {ExpectedLength}, got {value.Length} instead."); - } + var value = await device.ReadByteArrayAsync(Address); + return new AssertionResult( + value.Length == ExpectedLength, + x => x ? + $"Length is {ExpectedLength} as expected." : + $"Expected length of register to be {ExpectedLength}, got {value.Length} instead."); } [HarpTest(Description = "Checks if the register value is 0, indicating it is likely not used.")] - public async Task AssertReturnsZero(string portName) + public async Task AssertReturnsZero(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var value = await device.ReadByteArrayAsync(Address); - string msg = value.All(x => x == 0) ? "Value of all bytes is 0. Register likely not being used." : $"Register returned a non-zero value: {BitConverter.ToString(value)}."; - return new Result( - value, - Status.Passed, - msg); - } + var value = await device.ReadByteArrayAsync(Address); + string msg = value.All(x => x == 0) ? "Value of all bytes is 0. Register likely not being used." : $"Register returned a non-zero value: {BitConverter.ToString(value)}."; + return new Result( + value, + Status.Passed, + msg); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs index 917ced9..e1b149a 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs @@ -7,74 +7,62 @@ internal class R_VERSION : Suite public override string Description => "Version Register Tests"; [HarpTest(Description = "Validates that Version register is readable.")] - public async Task IsReadable(string portName) + public async Task IsReadable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) + try { - try - { - await device.ReadByteArrayAsync(Version.Address); - return new AssertionResult(true, "Version is readable."); - } - catch (Exception ex) - { - return new ErrorResult(ex); - } + await device.ReadByteArrayAsync(Version.Address); + return new AssertionResult(true, "Version is readable."); + } + catch (Exception ex) + { + return new ErrorResult(ex); } } [HarpTest(Description = "Validates that Version register has exactly 32 bytes.")] - public async Task AssertLength(string portName) + public async Task AssertLength(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - return await RegisterHelpers.AssertReadableArrayAsync(device, Version.Address, Version.RegisterLength, "Version"); - } + return await RegisterHelpers.AssertReadableArrayAsync(device, Version.Address, Version.RegisterLength, "Version"); } [HarpTest(Description = "Validates that Version register is NOT writable.")] - public async Task IsNotWritable(string portName) + public async Task IsNotWritable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var req = HarpMessage.FromByte(Version.Address, MessageType.Write, new byte[Version.RegisterLength]); - var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); - return new AssertionResult( - rejected, - x => x - ? "Version register correctly rejected write." - : "Version register should NOT be writable."); - } + var req = HarpMessage.FromByte(Version.Address, MessageType.Write, new byte[Version.RegisterLength]); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x + ? "Version register correctly rejected write." + : "Version register should NOT be writable."); } [HarpTest(Description = "Reports the version information declared by the device.")] - public async Task ReportVersionInformation(string portName) + public async Task ReportVersionInformation(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) + try { - try - { - var reply = await device.CommandAsync(HarpCommand.ReadByte(Version.Address)); - var payload = reply.GetPayloadArray(); - if (payload.Length != Version.RegisterLength) - { - return new AssertionResult( - false, - $"Version returned {payload.Length} bytes, expected {Version.RegisterLength}."); - } - - var version = Version.GetPayload(reply); - return new Result( - version, - Status.Passed, - $"PROTOCOL {version.ProtocolVersion}, FIRMWARE {version.FirmwareVersion}, " + - $"HARDWARE {version.HardwareVersion}, CORE_ID {version.CoreId}, " + - $"INTERFACE_HASH {Convert.ToHexString(version.InterfaceHash)}."); - } - catch (Exception ex) + var reply = await device.CommandAsync(HarpCommand.ReadByte(Version.Address)); + var payload = reply.GetPayloadArray(); + if (payload.Length != Version.RegisterLength) { - return new ErrorResult(ex); + return new AssertionResult( + false, + $"Version returned {payload.Length} bytes, expected {Version.RegisterLength}."); } + + var version = Version.GetPayload(reply); + return new Result( + version, + Status.Passed, + $"PROTOCOL {version.ProtocolVersion}, FIRMWARE {version.FirmwareVersion}, " + + $"HARDWARE {version.HardwareVersion}, CORE_ID {version.CoreId}, " + + $"INTERFACE_HASH {Convert.ToHexString(version.InterfaceHash)}."); + } + catch (Exception ex) + { + return new ErrorResult(ex); } } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs index 851376c..ffb216e 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_WHO_AM_I.cs @@ -7,30 +7,24 @@ internal class R_WHO_AM_I : Suite public override string Description => "WhoAmI Register Tests"; [HarpTest(Description = "Validates that the WhoAmI register exists and contains a value.")] - public async Task CheckWhoAmI(string portName) + public async Task CheckWhoAmI(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - int value = await device.ReadWhoAmIAsync(); - return new Result( - value, - (v) => v > 0 && v < 9999, - (v, success) => success ? $"WhoAmI register contains valid value: {v}." : $"WhoAmI register contains invalid value: {v}."); - } + int value = await device.ReadWhoAmIAsync(); + return new Result( + value, + (v) => v > 0 && v < 9999, + (v, success) => success ? $"WhoAmI register contains valid value: {v}." : $"WhoAmI register contains invalid value: {v}."); } [HarpTest(Description = "Validates that the WhoAmI register is NOT writable.")] - public async Task IsNotWritable(string portName) + public async Task IsNotWritable(VerifyConnection device) { - using (var device = new AsyncDevice(portName)) - { - var req = HarpMessage.FromUInt16(WhoAmI.Address, MessageType.Write, 0); - var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); - return new AssertionResult( - rejected, - x => x ? - "WhoAmI register correctly rejected write." : - "WhoAmI register should NOT be writable."); - } + var req = HarpMessage.FromUInt16(WhoAmI.Address, MessageType.Write, 0); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); + return new AssertionResult( + rejected, + x => x ? + "WhoAmI register correctly rejected write." : + "WhoAmI register should NOT be writable."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/RegisterHelpers.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/RegisterHelpers.cs index 2d9113c..183c644 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/RegisterHelpers.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/RegisterHelpers.cs @@ -1,49 +1,10 @@ - -using Bonsai.Harp; -using System.Reactive.Linq; -using System.Reactive.Subjects; +using Bonsai.Harp; namespace Harp.Toolkit.Verify.Suites; internal static class RegisterHelpers { - /// - /// Opens a Device connection, writes messages via the synchronous transport, - /// collects all received messages for the specified duration, then cleans up. - /// - public static async Task> WriteToTransportAsync( - string portName, - IEnumerable messagesToWrite, - TimeSpan listenDuration, - Action? configureDevice = null) - { - var harpDevice = new Bonsai.Harp.Device { PortName = portName }; - configureDevice?.Invoke(harpDevice); - - var source = new Subject(); - var collected = new List(); - var tcs = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - - using var subscription = harpDevice.Generate(source) - .Subscribe( - onNext: m => collected.Add(m), - onError: ex => tcs.TrySetException(ex)); - - // Small delay to let the transport connect - await Task.Delay(200); - - foreach (var msg in messagesToWrite) - { - source.OnNext(msg); - } - - await Task.Delay(listenDuration); - - source.OnCompleted(); - tcs.TrySetResult(collected); - return await tcs.Task; - } - public static async Task IsWriteRejectedAsync(AsyncDevice device, HarpMessage write) + public static async Task IsWriteRejectedAsync(VerifyConnection device, HarpMessage write) { try { @@ -56,7 +17,7 @@ public static async Task IsWriteRejectedAsync(AsyncDevice device, HarpMess } } - public static async Task AssertReadableArrayAsync(AsyncDevice device, int address, int expectedLength, string registerName) + public static async Task AssertReadableArrayAsync(VerifyConnection device, int address, int expectedLength, string registerName) { try { diff --git a/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs b/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs index acee796..10ce7f3 100644 --- a/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs @@ -1,7 +1,6 @@ using System.Reflection; using Bonsai.Harp; using Harp.Generators; -using Harp.Toolkit.Verify; namespace Harp.Toolkit.Verify.Suites; @@ -62,7 +61,7 @@ public DeviceInterfaceSuite(DeviceMetadata? metadata, string? rawYaml) "Validates the C# interface generated from device.yml by parsing live register replies with its own generated parsers, and cross-checks WhoAmI/firmware/hardware versions."; [HarpTest(Description = "Generates and compiles the C# interface from device.yml.")] - public Task GenerateAndCompileInterface(string portName) + public Task GenerateAndCompileInterface(VerifyConnection device) { IResult result = metadata is null ? new Result(false, Status.Skipped, "No device.yml provided (--device-yml).") @@ -73,12 +72,11 @@ public Task GenerateAndCompileInterface(string portName) } [HarpTest(Description = "Compares the WhoAmI/firmware/hardware version reported by the device against device.yml.")] - public async Task DeviceIdentity(string portName) + public async Task DeviceIdentity(VerifyConnection device) { if (metadata is null) return new Result(false, Status.Skipped, "No device.yml provided (--device-yml)."); - using var device = new AsyncDevice(portName); var mismatches = new List(); int whoAmI = await device.ReadWhoAmIAsync(); @@ -111,11 +109,11 @@ private static IReadOnlyList BuildRegisterTests(IReadOnlyDictionary .Select(entry => new DynamicTest( entry.Value.Name, $"Reads register '{entry.Value.Name}' (address {entry.Key}) and parses the reply with its generated GetPayload parser.", - (portName, cancellationToken) => CheckRegisterAsync(entry.Key, entry.Value, portName, cancellationToken))) + (device, cancellationToken) => CheckRegisterAsync(entry.Key, entry.Value, device, cancellationToken))) .ToList(); } - private static async Task CheckRegisterAsync(int address, Type registerType, string portName, CancellationToken cancellationToken) + private static async Task CheckRegisterAsync(int address, Type registerType, VerifyConnection device, CancellationToken cancellationToken) { const BindingFlags staticMembers = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static; @@ -125,7 +123,6 @@ private static async Task CheckRegisterAsync(int address, Type register var payloadType = (PayloadType)registerType.GetField("RegisterType", staticMembers)!.GetValue(null)!; - using var device = new AsyncDevice(portName); HarpMessage reply; try { diff --git a/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs b/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs index 27c0c2a..50c1a15 100644 --- a/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs @@ -13,18 +13,15 @@ public RoundTripTestSuite(double maxRoundTripDelayMs = 4.0) public override string Description => "A bunch of tests to benchmark round trip read/writes."; [HarpTest(Description = "Benchmarks the round trip time for a WhoAmI read command.")] - public async Task BenchmarkRoundTrip(string portName) + public async Task BenchmarkRoundTrip(VerifyConnection device) { const int n = 1000; double[] timestamps = new double[n]; HarpMessage probe = Bonsai.Harp.WhoAmI.FromPayload(MessageType.Read, default); - using (var device = new AsyncDevice(portName)) + for (int i = 0; i < n; i++) { - for (int i = 0; i < n; i++) - { - var reply = await device.CommandAsync(probe); - timestamps[i] = reply.GetTimestamp(); - } + var reply = await device.CommandAsync(probe); + timestamps[i] = reply.GetTimestamp(); } var derivatives = timestamps .Zip(timestamps.Skip(1), (previous, current) => (current - previous) * 1e3) diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index 4b88aad..4095d04 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -1,11 +1,10 @@ using System.CommandLine; using Spectre.Console; using Harp.Generators; -using Harp.Toolkit.Verify; using Harp.Toolkit.Verify.Suites; using Harp.Toolkit.Generate; -namespace Harp.Toolkit; +namespace Harp.Toolkit.Verify; public class VerifyCommand : Command { public VerifyCommand() @@ -100,8 +99,10 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve RunDate = DateTime.Now }; + using var connection = await VerifyConnection.OpenAsync(portName, cancellationToken); + int currentTest = 0; - await foreach (var (suite, result) in runner.RunAllAsync(portName, cancellationToken, (suite, testName, testDesc) => + await foreach (var (suite, result) in runner.RunAllAsync(connection, cancellationToken, (suite, testName, testDesc) => { // Print "Running" status before test execution (without newline) currentTest++; diff --git a/src/Harp.Toolkit/Verify/VerifyConnection.cs b/src/Harp.Toolkit/Verify/VerifyConnection.cs new file mode 100644 index 0000000..5cd021d --- /dev/null +++ b/src/Harp.Toolkit/Verify/VerifyConnection.cs @@ -0,0 +1,222 @@ +using System.Reactive.Linq; +using System.Reactive.Subjects; +using Bonsai.Harp; + +namespace Harp.Toolkit.Verify; + +/// +/// A single Harp connection shared by every test in a verification run. +/// +public sealed class VerifyConnection : IDisposable +{ + const int ConnectDelayMilliseconds = 200; + const int PortReleaseDelayMilliseconds = 300; + const int OpenTimeoutMilliseconds = 10000; + const int OpenRetryDelayMilliseconds = 250; + const int ReadyAttempts = 5; + const int ReadyTimeoutMilliseconds = 1000; + + readonly Subject requests = new(); + readonly IConnectableObservable messages; + readonly IDisposable subscription; + + VerifyConnection(string portName, int whoAmI) + { + var device = new Bonsai.Harp.Device(whoAmI) { PortName = portName, IgnoreErrors = true }; + messages = device.Generate(requests).Publish(); + subscription = messages.Connect(); + } + + /// + /// Opens the shared connection, reading the device identity first so the connection can be + /// constructed with a known WhoAmI, which suppresses the device name probe that would + /// otherwise open a second port in the background. + /// + public static async Task OpenAsync(string portName, CancellationToken cancellationToken = default) + { + var deadline = Environment.TickCount64 + OpenTimeoutMilliseconds; + var whoAmI = await ReadIdentityAsync(portName, deadline, cancellationToken); + await Task.Delay(PortReleaseDelayMilliseconds, cancellationToken); + + while (true) + { + var connection = new VerifyConnection(portName, whoAmI); + try + { + await connection.WaitUntilReadyAsync(cancellationToken); + return connection; + } + catch (UnauthorizedAccessException) when (Environment.TickCount64 < deadline && !cancellationToken.IsCancellationRequested) + { + connection.Dispose(); + await Task.Delay(OpenRetryDelayMilliseconds, cancellationToken); + } + catch + { + connection.Dispose(); + throw; + } + } + } + + static async Task ReadIdentityAsync(string portName, long deadline, CancellationToken cancellationToken) + { + while (true) + { + try + { + using var probe = new AsyncDevice(portName); + return await probe.ReadWhoAmIAsync(cancellationToken); + } + catch (UnauthorizedAccessException) when (Environment.TickCount64 < deadline && !cancellationToken.IsCancellationRequested) + { + await Task.Delay(OpenRetryDelayMilliseconds, cancellationToken); + } + } + } + + /// + /// Every message received from the device, before any reply correlation. + /// + public IObservable Messages => messages; + + /// + /// Sends a message without awaiting a reply. + /// + public void Write(HarpMessage message) => requests.OnNext(message); + + public async Task CommandAsync(HarpMessage command, CancellationToken cancellationToken = default) + { + var reply = messages.FirstAsync(message => + { + var match = message.IsMatch(command.Address, command.MessageType); + if (match && message.Error) + { + throw new HarpException(message); + } + + return match; + }).RunAsync(cancellationToken); + + Write(command); + return await reply; + } + + /// + /// Sends the specified messages and collects everything received for the given duration. + /// + public async Task> WriteAndCollectAsync( + IEnumerable messagesToWrite, + TimeSpan listenDuration, + CancellationToken cancellationToken = default) + { + var collected = new List(); + using (messages.Subscribe(message => + { + lock (collected) + { + collected.Add(message); + } + })) + { + foreach (var message in messagesToWrite) + { + Write(message); + } + + await Task.Delay(listenDuration, cancellationToken); + } + + lock (collected) + { + return collected.ToList(); + } + } + + public async Task ReadByteAsync(int address, CancellationToken cancellationToken = default) + { + var reply = await CommandAsync(HarpCommand.ReadByte(address), cancellationToken); + return reply.GetPayloadByte(); + } + + public async Task ReadByteArrayAsync(int address, CancellationToken cancellationToken = default) + { + var reply = await CommandAsync(HarpCommand.ReadByte(address), cancellationToken); + return reply.GetPayloadArray(); + } + + public async Task ReadUInt16Async(int address, CancellationToken cancellationToken = default) + { + var reply = await CommandAsync(HarpCommand.ReadUInt16(address), cancellationToken); + return reply.GetPayloadUInt16(); + } + + public async Task ReadUInt32Async(int address, CancellationToken cancellationToken = default) + { + var reply = await CommandAsync(HarpCommand.ReadUInt32(address), cancellationToken); + return reply.GetPayloadUInt32(); + } + + public async Task ReadWhoAmIAsync(CancellationToken cancellationToken = default) + { + return await ReadUInt16Async(WhoAmI.Address, cancellationToken); + } + + public async Task ReadAssemblyVersionAsync(CancellationToken cancellationToken = default) + { + return await ReadByteAsync(AssemblyVersion.Address, cancellationToken); + } + + public async Task ReadSerialNumberAsync(CancellationToken cancellationToken = default) + { + return await ReadUInt16Async(SerialNumber.Address, cancellationToken); + } + + public async Task ReadHardwareVersionAsync(CancellationToken cancellationToken = default) + { + var major = await ReadByteAsync(HardwareVersionHigh.Address, cancellationToken); + var minor = await ReadByteAsync(HardwareVersionLow.Address, cancellationToken); + return new HarpVersion(major, minor); + } + + public async Task ReadFirmwareVersionAsync(CancellationToken cancellationToken = default) + { + var major = await ReadByteAsync(FirmwareVersionHigh.Address, cancellationToken); + var minor = await ReadByteAsync(FirmwareVersionLow.Address, cancellationToken); + return new HarpVersion(major, minor); + } + + public async Task ReadTimestampSecondsAsync(CancellationToken cancellationToken = default) + { + return await ReadUInt32Async(TimestampSeconds.Address, cancellationToken); + } + + public async Task WriteTimestampSecondsAsync(uint seconds, CancellationToken cancellationToken = default) + { + await CommandAsync(HarpCommand.WriteUInt32(TimestampSeconds.Address, seconds), cancellationToken); + } + + async Task WaitUntilReadyAsync(CancellationToken cancellationToken) + { + await Task.Delay(ConnectDelayMilliseconds, cancellationToken); + for (int attempt = 1; ; attempt++) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(ReadyTimeoutMilliseconds); + try + { + await ReadWhoAmIAsync(timeout.Token); + return; + } + catch (OperationCanceledException) when (attempt < ReadyAttempts && !cancellationToken.IsCancellationRequested) + { + } + } + } + + public void Dispose() + { + subscription.Dispose(); + requests.Dispose(); + } +} From a20a74f8649e0f3105b4664af73c8244b44d901f Mon Sep 17 00:00:00 2001 From: glopesdev Date: Fri, 4 Sep 2026 08:15:51 +0100 Subject: [PATCH 25/41] Report round trip latency without asserting The benchmark now times each command with a stopwatch instead of differencing consecutive device reply timestamps, removing the dependency on the internal timestamp resolution of the device, which the protocol quantizes to 32 microseconds. It reports the resulting statistics and always passes. Mean, median, standard deviation, minimum, maximum and percentiles still appear in the console table and the HTML report. --- .../Verify/Suites/RoundTripTestSuite.cs | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs b/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs index 50c1a15..ff98f27 100644 --- a/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs @@ -1,39 +1,27 @@  +using System.Diagnostics; using Bonsai.Harp; namespace Harp.Toolkit.Verify.Suites; internal class RoundTripTestSuite : Suite { - private double maxRoundTripDelayMs; - public RoundTripTestSuite(double maxRoundTripDelayMs = 4.0) - { - this.maxRoundTripDelayMs = maxRoundTripDelayMs; - } - public override string Description => "A bunch of tests to benchmark round trip read/writes."; [HarpTest(Description = "Benchmarks the round trip time for a WhoAmI read command.")] public async Task BenchmarkRoundTrip(VerifyConnection device) { const int n = 1000; - double[] timestamps = new double[n]; + double[] elapsed = new double[n]; HarpMessage probe = Bonsai.Harp.WhoAmI.FromPayload(MessageType.Read, default); + var clock = new Stopwatch(); for (int i = 0; i < n; i++) { - var reply = await device.CommandAsync(probe); - timestamps[i] = reply.GetTimestamp(); - } - var derivatives = timestamps - .Zip(timestamps.Skip(1), (previous, current) => (current - previous) * 1e3) - .ToArray(); - var benchmark = new BenchmarkSummary(derivatives); - if (benchmark.Max > maxRoundTripDelayMs) - { - return new NumericBenchmarkResult(benchmark, Status.Failed, $"Round trip WhoAmI read benchmark exceeded maximum allowed delay of {maxRoundTripDelayMs} ms."); - } - else - { - return new NumericBenchmarkResult(benchmark, Status.Passed, "Round trip WhoAmI read benchmark."); + clock.Restart(); + await device.CommandAsync(probe); + elapsed[i] = clock.Elapsed.TotalMilliseconds; } + var benchmark = new BenchmarkSummary(elapsed); + return new NumericBenchmarkResult(benchmark, Status.Passed, + $"Round trip WhoAmI read latency over {n} samples, in milliseconds."); } } From 30ead1d2f780bd0953abbca288b9ab53dee8e313 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Fri, 4 Sep 2026 09:29:00 +0100 Subject: [PATCH 26/41] Skip operation control restore when read fails A failed read of the register would leave the restore writing zero, putting the device in Standby with every flag cleared. The original value is now nullable and the restore is skipped when it is absent. --- .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs index 22de604..423025e 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -53,7 +53,7 @@ public async Task VisualEnWritable(VerifyConnection device) [HarpTest(Description = "Validates that enabling HEARTBEAT_EN causes the device to emit R_HEARTBEAT events.")] public async Task HeartbeatEnEmitsEvents(VerifyConnection device) { - byte originalOpCtrl = 0; + byte? originalOpCtrl = null; try { @@ -83,7 +83,7 @@ public async Task HeartbeatEnEmitsEvents(VerifyConnection device) [HarpTest(Description = "Validates that HEARTBEAT_EN (bit 2) takes precedence over ALIVE_EN (bit 7): when both are set, R_HEARTBEAT events are emitted and R_TIMESTAMP_SECOND events are not.")] public async Task HeartbeatEnPrecedenceOverAliveEn(VerifyConnection device) { - byte originalOpCtrl = 0; + byte? originalOpCtrl = null; try { @@ -116,7 +116,7 @@ public async Task HeartbeatEnPrecedenceOverAliveEn(VerifyConnection dev [HarpTest(Description = "Validates that ALIVE_EN (deprecated, bit 7) causes R_TIMESTAMP_SECOND events to be emitted when HEARTBEAT_EN is not set.")] public async Task AliveEnEmitsTimestampEvents(VerifyConnection device) { - byte originalOpCtrl = 0; + byte? originalOpCtrl = null; try { @@ -146,14 +146,13 @@ public async Task AliveEnEmitsTimestampEvents(VerifyConnection device) [HarpTest(Description = "Validates that the DUMP bit triggers a burst of all core register reads after an OpCtrl write.")] public async Task RegisterDump(VerifyConnection device) { - byte originalOpCtrl = 0; + byte? originalOpCtrl = null; try { - // Read original state before modifying originalOpCtrl = await device.ReadByteAsync(OperationControl.Address); var messages = await device.WriteAndCollectAsync( - new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, (byte)(originalOpCtrl | 0x08)) }, + new[] { HarpMessage.FromByte(OperationControl.Address, MessageType.Write, (byte)(originalOpCtrl.GetValueOrDefault() | 0x08)) }, TimeSpan.FromSeconds(1)); var opRegWriteResponse = messages.FirstOrDefault(m => m.Address == OperationControl.Address && m.MessageType == MessageType.Write); @@ -183,11 +182,14 @@ public async Task RegisterDump(VerifyConnection device) } } - private static async Task RestoreOperationControlAsync(VerifyConnection device, byte value) + private static async Task RestoreOperationControlAsync(VerifyConnection device, byte? value) { + if (!value.HasValue) + return; + try { - await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, value)); + await device.CommandAsync(HarpMessage.FromByte(OperationControl.Address, MessageType.Write, value.GetValueOrDefault())); } catch { From 46bb218ff3320a416895849de4c02d5eab73ecb1 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Fri, 4 Sep 2026 10:20:51 +0100 Subject: [PATCH 27/41] Remove unused stopwatch from timestamp test --- .../Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs index a326be9..89683fd 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs @@ -1,6 +1,5 @@  using Bonsai.Harp; -using System.Diagnostics; namespace Harp.Toolkit.Verify.Suites; internal class R_TIMESTAMP_SECOND : Suite @@ -50,7 +49,6 @@ public async Task IsMonotonic(VerifyConnection device) [HarpTest(Description = "Validates that writing a past timestamp value takes effect and can be read back.")] public async Task WritePastValueRoundTrip(VerifyConnection device) { - var sw = Stopwatch.StartNew(); var current = await device.ReadTimestampSecondsAsync(); var tPast = current >= 10 ? current - 10 : 0u; From fc496ba0a5ad67ad9477142a794e50e2306318b4 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sun, 6 Sep 2026 22:56:25 +0100 Subject: [PATCH 28/41] Add boot provenance and read-only bit tests The reset device suite now checks that a read reports exactly one boot provenance bit with every command bit cleared, so the only legal values are 0x40 and 0x80, and that writes setting BOOT_DEF or BOOT_EE are refused, since both are read-only state. Both write probes fail on current ATxmega firmware, which answers them with a write reply rather than an error. --- .../Suites/CoreRegisters/R_RESET_DEV.cs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs index af08d03..ad4a245 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs @@ -4,6 +4,8 @@ namespace Harp.Toolkit.Verify.Suites; internal class R_RESET_DEV : Suite { + const ResetFlags BootProvenance = ResetFlags.BootFromDefault | ResetFlags.BootFromEeprom; + public override string Description => "Reset Device Register Tests"; [HarpTest(Description = "Validates that ResetDev register is readable.")] @@ -11,4 +13,72 @@ public async Task IsReadable(VerifyConnection device) { return await RegisterHelpers.AssertReadableAsync(a => device.ReadByteAsync(a), ResetDevice.Address, "ResetDev"); } + + [HarpTest(Description = "Validates that a read of ResetDev clears every command bit and reports exactly one boot provenance bit.")] + public async Task AssertBootProvenanceReported(VerifyConnection device) + { + var value = await device.ReadByteAsync(ResetDevice.Address); + var flags = (ResetFlags)value; + return new AssertionResult( + flags == ResetFlags.BootFromDefault || flags == ResetFlags.BootFromEeprom, + x => x + ? $"ResetDev read 0x{value:X2}, reporting {DescribeBootSource(flags)}." + : $"ResetDev read 0x{value:X2}. {DescribeReadViolation(flags)}"); + } + + [HarpTest(Description = "Validates that ResetDev rejects a write setting BOOT_DEF, which is read-only state.")] + public async Task BootFromDefaultIsNotWritable(VerifyConnection device) + { + return await AssertReadOnlyBitRejectedAsync(device, ResetFlags.BootFromDefault, "BOOT_DEF"); + } + + [HarpTest(Description = "Validates that ResetDev rejects a write setting BOOT_EE, which is read-only state.")] + public async Task BootFromEepromIsNotWritable(VerifyConnection device) + { + return await AssertReadOnlyBitRejectedAsync(device, ResetFlags.BootFromEeprom, "BOOT_EE"); + } + + static async Task AssertReadOnlyBitRejectedAsync(VerifyConnection device, ResetFlags bit, string bitName) + { + var request = HarpMessage.FromByte(ResetDevice.Address, MessageType.Write, (byte)bit); + var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, request); + return new AssertionResult( + rejected, + x => x + ? $"ResetDev correctly rejected a write setting {bitName}." + : $"ResetDev accepted a write setting {bitName}, which is read-only state and must be answered with an error reply."); + } + + static string DescribeBootSource(ResetFlags flags) + { + return flags == ResetFlags.BootFromEeprom + ? "a boot from register values stored in non-volatile memory" + : "a boot from default register values"; + } + + static string DescribeReadViolation(ResetFlags flags) + { + var provenance = flags & BootProvenance; + if (provenance == BootProvenance) + return "Both BOOT_DEF and BOOT_EE are set, so the reported boot provenance is contradictory."; + if (provenance == 0) + return "Neither BOOT_DEF nor BOOT_EE is set, so no boot provenance is reported. A device without non-volatile memory must always set BOOT_DEF."; + + var commandBits = DescribeCommandBits(flags); + if (commandBits.Length > 0) + return $"A read reply must clear every command bit, but {commandBits} remained set."; + + var remaining = (byte)(flags & ~BootProvenance); + return $"A read reply must clear every bit outside the boot provenance field, but 0x{remaining:X2} remained set."; + } + + static string DescribeCommandBits(ResetFlags flags) + { + var names = new List(); + if (flags.HasFlag(ResetFlags.RestoreDefault)) names.Add("RST_DEF"); + if (flags.HasFlag(ResetFlags.RestoreEeprom)) names.Add("RST_EE"); + if (flags.HasFlag(ResetFlags.Save)) names.Add("SAVE"); + if (flags.HasFlag(ResetFlags.RestoreName)) names.Add("NAME_TO_DEFAULT"); + return string.Join(", ", names); + } } From c732aaff96805229c3c7f5de3d9e5d3521a2515a Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sun, 6 Sep 2026 22:57:51 +0100 Subject: [PATCH 29/41] Check register dump against declared core schema The register dump test now resolves the expected addresses and payload types from the core register metadata embedded in Harp.Generators, rather than from a hardcoded range of twenty addresses. It requires the fifteen declared addresses, ignores any beyond them, and now also fails when a reply carries a payload type the schema does not declare. The generators reference moves to 0.7.0, which is what pins the schema the test reads. --- src/Harp.Toolkit/Harp.Toolkit.csproj | 2 +- src/Harp.Toolkit/Verify/CoreSchema.cs | 28 ++++++++++++++++++ .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 29 ++++++++++++++----- 3 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 src/Harp.Toolkit/Verify/CoreSchema.cs diff --git a/src/Harp.Toolkit/Harp.Toolkit.csproj b/src/Harp.Toolkit/Harp.Toolkit.csproj index 571acd5..bbfa6ee 100644 --- a/src/Harp.Toolkit/Harp.Toolkit.csproj +++ b/src/Harp.Toolkit/Harp.Toolkit.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/Harp.Toolkit/Verify/CoreSchema.cs b/src/Harp.Toolkit/Verify/CoreSchema.cs new file mode 100644 index 0000000..56d1ac2 --- /dev/null +++ b/src/Harp.Toolkit/Verify/CoreSchema.cs @@ -0,0 +1,28 @@ +using Harp.Generators; + +namespace Harp.Toolkit.Verify; + +/// +/// Provides the core register metadata embedded in the Harp.Generators assembly, which declares +/// the register set and payload types a conformant device must implement. +/// +internal static class CoreSchema +{ + const string ResourceName = "Harp.Generators.core.yml"; + + static readonly Lazy metadata = new(ReadMetadata); + + /// + /// Gets the core register metadata declared by the pinned generator version. + /// + public static DeviceMetadata Metadata => metadata.Value; + + static DeviceMetadata ReadMetadata() + { + using var stream = typeof(InterfaceGenerator).Assembly.GetManifestResourceStream(ResourceName) + ?? throw new InvalidOperationException( + $"The core register metadata resource '{ResourceName}' was not found in the Harp.Generators assembly."); + using var reader = new StreamReader(stream); + return MetadataDeserializer.Instance.Deserialize(reader); + } +} diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs index 423025e..1e4b3e1 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -160,17 +160,32 @@ public async Task RegisterDump(VerifyConnection device) { return new AssertionResult(false, "No response received for OpCtrl write."); } - var coreReads = messages - .Select((m, i) => (msg: m, idx: i)) - .Where(x => x.msg.Address < 32 && x.msg.MessageType == MessageType.Read) + var replies = messages + .Where(m => m.MessageType == MessageType.Read) + .GroupBy(m => m.Address) + .ToDictionary(g => g.Key, g => g.First()); + + var declared = CoreSchema.Metadata.Registers.Values; + var missing = declared + .Where(r => !replies.ContainsKey(r.Address)) + .Select(r => r.Address) + .OrderBy(a => a) .ToList(); - var uniqueCoreAddresses = coreReads.Select(x => x.msg.Address).Distinct().ToHashSet(); - var missing = Enumerable.Range(0, 20).Where(a => !uniqueCoreAddresses.Contains(a)).ToList(); if (missing.Count > 0) return new AssertionResult(false, - $"Missing Read replies for {missing.Count} core address(es): {string.Join(", ", missing)}."); + $"Missing Read replies for {missing.Count} declared core address(es): {string.Join(", ", missing)}."); - return new AssertionResult(true, "All core register reads received after OpCtrl write."); + var mismatched = declared + .Select(r => (Register: r, Replied: replies[r.Address].PayloadType & ~PayloadType.Timestamp)) + .Where(x => x.Replied != x.Register.Type) + .Select(x => $"address {x.Register.Address} declares {x.Register.Type} but replied {x.Replied}") + .ToList(); + if (mismatched.Count > 0) + return new AssertionResult(false, + $"Payload type mismatch on {mismatched.Count} core register(s): {string.Join("; ", mismatched)}."); + + return new AssertionResult(true, + $"All {declared.Count} declared core registers were dumped with the expected payload type."); } catch (Exception ex) { From dbb18804d3e8b4df6aea71dd1b812ea289768f28 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 7 Sep 2026 13:56:30 +0100 Subject: [PATCH 30/41] Restrict prerelease checks behind an opt-in flag Tests asserting specification text outside the stable baseline are marked with Prerelease on HarpTestAttribute and excluded during collection, so the progress counter and the report cover only what runs. The new --prerelease option includes them. Twenty-two of the fifty-one tests are marked, and the Behavior device now passes the whole default run. --- src/Harp.Toolkit/Verify/HarpTestAttribute.cs | 2 ++ src/Harp.Toolkit/Verify/Report.cs | 2 ++ src/Harp.Toolkit/Verify/ReportTemplate.cshtml | 16 ++++++++++ src/Harp.Toolkit/Verify/Runner.cs | 10 +++++-- src/Harp.Toolkit/Verify/Suite.cs | 15 +++++++--- .../Suites/CoreRegisters/R_CORE_VERSION_H.cs | 2 +- .../Suites/CoreRegisters/R_CORE_VERSION_L.cs | 2 +- .../Suites/CoreRegisters/R_FW_VERSION_H.cs | 2 +- .../Suites/CoreRegisters/R_FW_VERSION_L.cs | 2 +- .../Suites/CoreRegisters/R_HEARTBEAT.cs | 4 +-- .../Suites/CoreRegisters/R_HW_VERSION_H.cs | 2 +- .../Suites/CoreRegisters/R_HW_VERSION_L.cs | 2 +- .../Suites/CoreRegisters/R_RESET_DEV.cs | 4 +-- .../Suites/CoreRegisters/R_SERIAL_NUMBER.cs | 2 +- .../Verify/Suites/CoreRegisters/R_TAG.cs | 6 ++-- .../CoreRegisters/R_TIMESTAMP_OFFSET.cs | 4 +-- .../Verify/Suites/CoreRegisters/R_UID.cs | 4 +-- .../Verify/Suites/CoreRegisters/R_VERSION.cs | 8 ++--- src/Harp.Toolkit/Verify/VerifyCommand.cs | 29 +++++++++++++++---- 19 files changed, 84 insertions(+), 34 deletions(-) diff --git a/src/Harp.Toolkit/Verify/HarpTestAttribute.cs b/src/Harp.Toolkit/Verify/HarpTestAttribute.cs index 701d5b6..42b091d 100644 --- a/src/Harp.Toolkit/Verify/HarpTestAttribute.cs +++ b/src/Harp.Toolkit/Verify/HarpTestAttribute.cs @@ -4,4 +4,6 @@ public class HarpTestAttribute : Attribute { public string? Description { get; set; } + + public bool Prerelease { get; set; } } diff --git a/src/Harp.Toolkit/Verify/Report.cs b/src/Harp.Toolkit/Verify/Report.cs index 3e387d2..1f7f1ee 100644 --- a/src/Harp.Toolkit/Verify/Report.cs +++ b/src/Harp.Toolkit/Verify/Report.cs @@ -4,5 +4,7 @@ public class Report { public string DeviceName { get; set; } = "Unknown Device"; public DateTime RunDate { get; set; } = DateTime.Now; + public bool IncludePrerelease { get; set; } + public int PrereleaseTestCount { get; set; } public List Suites { get; set; } = new(); } diff --git a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml index 0b9b66e..419d4c0 100644 --- a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml +++ b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml @@ -32,6 +32,22 @@ + @if (Model.PrereleaseTestCount > 0) + { + @if (Model.IncludePrerelease) + { + + } + else + { + + } + } + @foreach (var suite in Model.Suites) {
diff --git a/src/Harp.Toolkit/Verify/Runner.cs b/src/Harp.Toolkit/Verify/Runner.cs index 4d9484f..ed5ca46 100644 --- a/src/Harp.Toolkit/Verify/Runner.cs +++ b/src/Harp.Toolkit/Verify/Runner.cs @@ -5,12 +5,16 @@ namespace Harp.Toolkit.Verify; public class Runner { private readonly List suites = new(); + private readonly bool includePrerelease; - public Runner() + public Runner(bool includePrerelease) { + this.includePrerelease = includePrerelease; } - public int TestCount => suites.Sum(s => s.TestCount); + public int TestCount => suites.Sum(s => s.GetTestCount(includePrerelease)); + + public int PrereleaseTestCount => suites.Sum(s => s.GetPrereleaseTestCount()); public IEnumerable CollectSuites() { @@ -21,7 +25,7 @@ public IEnumerable CollectSuites() { foreach (var suite in suites) { - await foreach (var result in suite.RunAllAsync(connection, cancellationToken, (testName, testDesc) => onTestStart?.Invoke(suite, testName, testDesc))) + await foreach (var result in suite.RunAllAsync(connection, includePrerelease, cancellationToken, (testName, testDesc) => onTestStart?.Invoke(suite, testName, testDesc))) { yield return (suite, result); } diff --git a/src/Harp.Toolkit/Verify/Suite.cs b/src/Harp.Toolkit/Verify/Suite.cs index ef41f58..3a8b871 100644 --- a/src/Harp.Toolkit/Verify/Suite.cs +++ b/src/Harp.Toolkit/Verify/Suite.cs @@ -16,9 +16,11 @@ public abstract class Suite ///
protected virtual IReadOnlyList DynamicTests { get; } = new List(); - public int TestCount => CollectTests().Count() + DynamicTests.Count; + public int GetTestCount(bool includePrerelease) => CollectTests(includePrerelease).Count() + DynamicTests.Count; - private IEnumerable<(MethodInfo Method, HarpTestAttribute Attribute)> CollectTests() + public int GetPrereleaseTestCount() => CollectAllTests().Count(x => x.Attribute.Prerelease); + + private IEnumerable<(MethodInfo Method, HarpTestAttribute Attribute)> CollectAllTests() { return GetType() .GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) @@ -26,9 +28,14 @@ public abstract class Suite .Where(x => x.Attribute != null); } - public async IAsyncEnumerable RunAllAsync(VerifyConnection connection, [EnumeratorCancellation] CancellationToken cancellationToken = default, Action? onTestStart = null) + private IEnumerable<(MethodInfo Method, HarpTestAttribute Attribute)> CollectTests(bool includePrerelease) + { + return CollectAllTests().Where(x => includePrerelease || !x.Attribute.Prerelease); + } + + public async IAsyncEnumerable RunAllAsync(VerifyConnection connection, bool includePrerelease, [EnumeratorCancellation] CancellationToken cancellationToken = default, Action? onTestStart = null) { - foreach (var (method, attr) in CollectTests()) + foreach (var (method, attr) in CollectTests(includePrerelease)) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs index 444a527..8126972 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_H.cs @@ -6,7 +6,7 @@ internal class R_CORE_VERSION_H : Suite { public override string Description => "Core Version High Register Tests"; - [HarpTest(Description = "Validates that CoreVersionHigh matches byte 0 of R_VERSION.")] + [HarpTest(Description = "Validates that CoreVersionHigh matches byte 0 of R_VERSION.", Prerelease = true)] public async Task AssertConsistentWithVersion(VerifyConnection device) { var versionArray = await device.ReadByteArrayAsync(Version.Address); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs index 635cbb1..cb20f61 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_CORE_VERSION_L.cs @@ -6,7 +6,7 @@ internal class R_CORE_VERSION_L : Suite { public override string Description => "Core Version Low Register Tests"; - [HarpTest(Description = "Validates that CoreVersionLow matches byte 1 of R_VERSION.")] + [HarpTest(Description = "Validates that CoreVersionLow matches byte 1 of R_VERSION.", Prerelease = true)] public async Task AssertConsistentWithVersion(VerifyConnection device) { var versionArray = await device.ReadByteArrayAsync(Version.Address); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs index 6f344d4..b6ac7ff 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_H.cs @@ -6,7 +6,7 @@ internal class R_FW_VERSION_H : Suite { public override string Description => "Firmware Version High Register Tests"; - [HarpTest(Description = "Validates that FwVersionHigh matches byte 3 of R_VERSION.")] + [HarpTest(Description = "Validates that FwVersionHigh matches byte 3 of R_VERSION.", Prerelease = true)] public async Task AssertConsistentWithVersion(VerifyConnection device) { var versionArray = await device.ReadByteArrayAsync(Version.Address); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs index 7b61f5f..1990d5b 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_FW_VERSION_L.cs @@ -6,7 +6,7 @@ internal class R_FW_VERSION_L : Suite { public override string Description => "Firmware Version Low Register Tests"; - [HarpTest(Description = "Validates that FwVersionLow matches byte 4 of R_VERSION.")] + [HarpTest(Description = "Validates that FwVersionLow matches byte 4 of R_VERSION.", Prerelease = true)] public async Task AssertConsistentWithVersion(VerifyConnection device) { var versionArray = await device.ReadByteArrayAsync(Version.Address); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs index 788a772..13c224c 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HEARTBEAT.cs @@ -7,13 +7,13 @@ internal class R_HEARTBEAT : Suite internal const byte Address = 18; public override string Description => "Heartbeat Register Tests"; - [HarpTest(Description = "Validates that Heartbeat register is readable.")] + [HarpTest(Description = "Validates that Heartbeat register is readable.", Prerelease = true)] public async Task IsReadable(VerifyConnection device) { return await RegisterHelpers.AssertReadableAsync(a => device.ReadUInt16Async(a), Address, "Heartbeat"); } - [HarpTest(Description = "Validates that Heartbeat register is NOT writable.")] + [HarpTest(Description = "Validates that Heartbeat register is NOT writable.", Prerelease = true)] public async Task IsNotWritable(VerifyConnection device) { var req = HarpMessage.FromUInt16(Address, MessageType.Write, 0); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs index c5a2581..9792179 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_H.cs @@ -6,7 +6,7 @@ internal class R_HW_VERSION_H : Suite { public override string Description => "Hardware Version High Register Tests"; - [HarpTest(Description = "Validates that HwVersionHigh matches byte 6 of R_VERSION.")] + [HarpTest(Description = "Validates that HwVersionHigh matches byte 6 of R_VERSION.", Prerelease = true)] public async Task AssertConsistentWithVersion(VerifyConnection device) { var versionArray = await device.ReadByteArrayAsync(Version.Address); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs index 85e9078..1468531 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_HW_VERSION_L.cs @@ -6,7 +6,7 @@ internal class R_HW_VERSION_L : Suite { public override string Description => "Hardware Version Low Register Tests"; - [HarpTest(Description = "Validates that HwVersionLow matches byte 7 of R_VERSION.")] + [HarpTest(Description = "Validates that HwVersionLow matches byte 7 of R_VERSION.", Prerelease = true)] public async Task AssertConsistentWithVersion(VerifyConnection device) { var versionArray = await device.ReadByteArrayAsync(Version.Address); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs index ad4a245..a1d01eb 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs @@ -26,13 +26,13 @@ public async Task AssertBootProvenanceReported(VerifyConnection device) : $"ResetDev read 0x{value:X2}. {DescribeReadViolation(flags)}"); } - [HarpTest(Description = "Validates that ResetDev rejects a write setting BOOT_DEF, which is read-only state.")] + [HarpTest(Description = "Validates that ResetDev rejects a write setting BOOT_DEF, which is read-only state.", Prerelease = true)] public async Task BootFromDefaultIsNotWritable(VerifyConnection device) { return await AssertReadOnlyBitRejectedAsync(device, ResetFlags.BootFromDefault, "BOOT_DEF"); } - [HarpTest(Description = "Validates that ResetDev rejects a write setting BOOT_EE, which is read-only state.")] + [HarpTest(Description = "Validates that ResetDev rejects a write setting BOOT_EE, which is read-only state.", Prerelease = true)] public async Task BootFromEepromIsNotWritable(VerifyConnection device) { return await AssertReadOnlyBitRejectedAsync(device, ResetFlags.BootFromEeprom, "BOOT_EE"); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs index a10843f..b6256b2 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs @@ -4,7 +4,7 @@ internal class R_SERIAL_NUMBER : Suite { public override string Description => "Serial Number Register Tests"; - [HarpTest(Description = "Validates that SerialNumber matches the first two bytes of R_UID.")] + [HarpTest(Description = "Validates that SerialNumber matches the first two bytes of R_UID.", Prerelease = true)] public async Task AssertConsistentWithUid(VerifyConnection device) { var uidValue = await device.ReadByteArrayAsync(R_UID.Address); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs index b452e95..fc776a4 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TAG.cs @@ -8,7 +8,7 @@ internal class R_TAG : Suite private const int ExpectedLength = 8; public override string Description => "Tag Register Tests"; - [HarpTest(Description = "Validates that Tag register is readable.")] + [HarpTest(Description = "Validates that Tag register is readable.", Prerelease = true)] public async Task IsReadable(VerifyConnection device) { try @@ -22,13 +22,13 @@ public async Task IsReadable(VerifyConnection device) } } - [HarpTest(Description = "Validates that Tag register has exactly 8 bytes.")] + [HarpTest(Description = "Validates that Tag register has exactly 8 bytes.", Prerelease = true)] public async Task AssertLength(VerifyConnection device) { return await RegisterHelpers.AssertReadableArrayAsync(device, Address, ExpectedLength, "Tag"); } - [HarpTest(Description = "Validates that Tag register is NOT writable.")] + [HarpTest(Description = "Validates that Tag register is NOT writable.", Prerelease = true)] public async Task IsNotWritable(VerifyConnection device) { var req = HarpMessage.FromByte(Address, MessageType.Write, new byte[ExpectedLength]); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs index 4b273ee..cf86d27 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs @@ -7,7 +7,7 @@ internal class R_TIMESTAMP_OFFSET : Suite private const byte Address = 15; public override string Description => "Timestamp Offset Register Tests"; - [HarpTest(Description = "Validates the deprecated register TimestampOffset returns 0x00.")] + [HarpTest(Description = "Validates the deprecated register TimestampOffset returns 0x00.", Prerelease = true)] public async Task AssertReturnsZero(VerifyConnection device) { var value = await device.ReadByteAsync(Address); @@ -18,7 +18,7 @@ public async Task AssertReturnsZero(VerifyConnection device) $"TimestampOffset register returned a non-zero value (0x{value:X2})."); } - [HarpTest(Description = "Validates the deprecated register TimestampOffset is NOT writable.")] + [HarpTest(Description = "Validates the deprecated register TimestampOffset is NOT writable.", Prerelease = true)] public async Task IsNotWritable(VerifyConnection device) { var req = HarpMessage.FromByte(Address, MessageType.Write, 0x00); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs index 2b193bd..bbaa994 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_UID.cs @@ -6,7 +6,7 @@ internal class R_UID : Suite private const byte ExpectedLength = 16; public override string Description => "UID Register Tests"; - [HarpTest(Description = "Validates that UID register has exactly 16 bytes.")] + [HarpTest(Description = "Validates that UID register has exactly 16 bytes.", Prerelease = true)] public async Task AssertLength(VerifyConnection device) { var value = await device.ReadByteArrayAsync(Address); @@ -17,7 +17,7 @@ public async Task AssertLength(VerifyConnection device) $"Expected length of register to be {ExpectedLength}, got {value.Length} instead."); } - [HarpTest(Description = "Checks if the register value is 0, indicating it is likely not used.")] + [HarpTest(Description = "Checks if the register value is 0, indicating it is likely not used.", Prerelease = true)] public async Task AssertReturnsZero(VerifyConnection device) { var value = await device.ReadByteArrayAsync(Address); diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs index e1b149a..25ff5eb 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs @@ -6,7 +6,7 @@ internal class R_VERSION : Suite { public override string Description => "Version Register Tests"; - [HarpTest(Description = "Validates that Version register is readable.")] + [HarpTest(Description = "Validates that Version register is readable.", Prerelease = true)] public async Task IsReadable(VerifyConnection device) { try @@ -20,13 +20,13 @@ public async Task IsReadable(VerifyConnection device) } } - [HarpTest(Description = "Validates that Version register has exactly 32 bytes.")] + [HarpTest(Description = "Validates that Version register has exactly 32 bytes.", Prerelease = true)] public async Task AssertLength(VerifyConnection device) { return await RegisterHelpers.AssertReadableArrayAsync(device, Version.Address, Version.RegisterLength, "Version"); } - [HarpTest(Description = "Validates that Version register is NOT writable.")] + [HarpTest(Description = "Validates that Version register is NOT writable.", Prerelease = true)] public async Task IsNotWritable(VerifyConnection device) { var req = HarpMessage.FromByte(Version.Address, MessageType.Write, new byte[Version.RegisterLength]); @@ -38,7 +38,7 @@ public async Task IsNotWritable(VerifyConnection device) : "Version register should NOT be writable."); } - [HarpTest(Description = "Reports the version information declared by the device.")] + [HarpTest(Description = "Reports the version information declared by the device.", Prerelease = true)] public async Task ReportVersionInformation(VerifyConnection device) { try diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index 4095d04..1675d7c 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -23,6 +23,12 @@ public VerifyCommand() Required = false, }; + Option prereleaseOption = new("--prerelease") + { + Description = "Include checks against specification text outside the stable baseline.", + Required = false, + }; + Option clockPortOption = new("--clock-port") { Description = "Serial port of the reference clock device. Enables clock alignment tests.", @@ -57,6 +63,7 @@ public VerifyCommand() Options.Add(portNameOption); Options.Add(fileOption); Options.Add(verboseOption); + Options.Add(prereleaseOption); Options.Add(clockPortOption); Options.Add(ppsEventOption); Options.Add(clockSamplesOption); @@ -66,17 +73,18 @@ public VerifyCommand() string portName = parsedResult.GetRequiredValue(portNameOption); FileInfo? reportFile = parsedResult.GetValue(fileOption); bool verbose = parsedResult.GetValue(verboseOption); + bool prerelease = parsedResult.GetValue(prereleaseOption); string? clockPort = parsedResult.GetValue(clockPortOption); ClockTestOptions? clockOptions = clockPort is null ? null : new ClockTestOptions( ClockPort: clockPort, PpsEvent: parsedResult.GetValue(ppsEventOption), ClockSamples: parsedResult.GetValue(clockSamplesOption)); FileInfo? deviceYml = parsedResult.GetValue(deviceYmlOption); - return RunVerification(portName, reportFile, verbose, clockOptions, deviceYml, CancellationToken.None); + return RunVerification(portName, reportFile, verbose, prerelease, clockOptions, deviceYml, CancellationToken.None); }); } - static async Task RunVerification(string portName, FileInfo? reportFile, bool verbose, ClockTestOptions? clockOptions, FileInfo? deviceYml, CancellationToken cancellationToken) + static async Task RunVerification(string portName, FileInfo? reportFile, bool verbose, bool prerelease, ClockTestOptions? clockOptions, FileInfo? deviceYml, CancellationToken cancellationToken) { AnsiConsole.MarkupLine($"Running tests on [bold]{portName}[/]..."); if (clockOptions is not null) @@ -92,11 +100,21 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve AnsiConsole.MarkupLine($" [green]Done![/] ({deviceMetadata.Registers.Count} registers)"); } - var runner = new CoreRunner(clockOptions, deviceMetadata, deviceRawYaml); + var runner = new CoreRunner(prerelease, clockOptions, deviceMetadata, deviceRawYaml); + if (runner.PrereleaseTestCount > 0) + { + if (prerelease) + AnsiConsole.MarkupLine($"Including [bold]{runner.PrereleaseTestCount}[/] prerelease checks."); + else + AnsiConsole.MarkupLine($"[yellow]{runner.PrereleaseTestCount} prerelease checks were not run. Rerun with --prerelease to include them.[/]"); + } + var report = new Report { DeviceName = $"Harp Device ({portName})", - RunDate = DateTime.Now + RunDate = DateTime.Now, + IncludePrerelease = prerelease, + PrereleaseTestCount = runner.PrereleaseTestCount }; using var connection = await VerifyConnection.OpenAsync(portName, cancellationToken); @@ -203,9 +221,10 @@ static string GetResultMarkup(IResult result) class CoreRunner : Runner { public CoreRunner( + bool includePrerelease, ClockTestOptions? clockOptions = null, DeviceMetadata? deviceMetadata = null, - string? deviceRawYaml = null) : base() + string? deviceRawYaml = null) : base(includePrerelease) { AddSuite(new R_WHO_AM_I()); AddSuite(new R_HW_VERSION_H()); From be4a1897fffdff48e777d6865b0b1cb1f82f9bf6 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 8 Sep 2026 09:54:42 +0100 Subject: [PATCH 31/41] Select the checked protocol version from R_VERSION Verification now reads the protocol version declared by the device and holds it to that revision. A device with no R_VERSION falls back to v1, and a version beyond coverage is reported before the run continues against v1. The --prerelease option also requires a device declaring v2, so on a v1 device it reports having no effect rather than silently including nothing. The console states the declared version and the revision applied, and the report header adds both plus the generator version supplying the register set. --- src/Harp.Toolkit/Verify/CoreSchema.cs | 23 +++++- src/Harp.Toolkit/Verify/ProtocolTarget.cs | 58 +++++++++++++ src/Harp.Toolkit/Verify/Report.cs | 5 +- src/Harp.Toolkit/Verify/ReportTemplate.cshtml | 30 ++++--- src/Harp.Toolkit/Verify/VerifyCommand.cs | 81 ++++++++++++++++--- 5 files changed, 172 insertions(+), 25 deletions(-) create mode 100644 src/Harp.Toolkit/Verify/ProtocolTarget.cs diff --git a/src/Harp.Toolkit/Verify/CoreSchema.cs b/src/Harp.Toolkit/Verify/CoreSchema.cs index 56d1ac2..76629b5 100644 --- a/src/Harp.Toolkit/Verify/CoreSchema.cs +++ b/src/Harp.Toolkit/Verify/CoreSchema.cs @@ -1,4 +1,5 @@ -using Harp.Generators; +using System.Reflection; +using Harp.Generators; namespace Harp.Toolkit.Verify; @@ -11,12 +12,32 @@ internal static class CoreSchema const string ResourceName = "Harp.Generators.core.yml"; static readonly Lazy metadata = new(ReadMetadata); + static readonly Lazy version = new(ReadVersion); /// /// Gets the core register metadata declared by the pinned generator version. /// public static DeviceMetadata Metadata => metadata.Value; + /// + /// Gets the version of the generator package supplying the core register metadata. This + /// version fully determines the register set, since the metadata ships inside the package. + /// + public static string Version => version.Value; + + static string ReadVersion() + { + var assembly = typeof(InterfaceGenerator).Assembly; + var informational = assembly + .GetCustomAttribute()?.InformationalVersion; + var text = informational ?? assembly.GetName().Version?.ToString(); + if (string.IsNullOrEmpty(text)) + return "unknown"; + + var metadataSeparator = text.IndexOf('+'); + return metadataSeparator < 0 ? text : text[..metadataSeparator]; + } + static DeviceMetadata ReadMetadata() { using var stream = typeof(InterfaceGenerator).Assembly.GetManifestResourceStream(ResourceName) diff --git a/src/Harp.Toolkit/Verify/ProtocolTarget.cs b/src/Harp.Toolkit/Verify/ProtocolTarget.cs new file mode 100644 index 0000000..6ab3ae7 --- /dev/null +++ b/src/Harp.Toolkit/Verify/ProtocolTarget.cs @@ -0,0 +1,58 @@ +using Bonsai.Harp; +using Harp.Toolkit.Verify.Suites; + +namespace Harp.Toolkit.Verify; + +internal enum ProtocolScope +{ + V1, + V2, + Unsupported, +} + +internal readonly record struct ProtocolTarget(SemanticVersion? DeclaredVersion, bool PrereleaseRequested) +{ + const int PrereleaseMajorVersion = 2; + const int ReadTimeoutMilliseconds = 2000; + + public ProtocolScope Scope + { + get + { + var major = DeclaredVersion.HasValue ? DeclaredVersion.GetValueOrDefault().Major : 0; + if (major > PrereleaseMajorVersion) + return ProtocolScope.Unsupported; + + return major == PrereleaseMajorVersion ? ProtocolScope.V2 : ProtocolScope.V1; + } + } + + public bool IncludePrerelease => Scope == ProtocolScope.V2 && PrereleaseRequested; + + public static async Task ResolveAsync( + VerifyConnection connection, + bool prereleaseRequested, + CancellationToken cancellationToken = default) + { + using var readTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + readTimeout.CancelAfter(ReadTimeoutMilliseconds); + try + { + var reply = await connection.CommandAsync( + HarpCommand.ReadByte(Suites.Version.Address), + readTimeout.Token); + var payload = reply.GetPayloadArray(); + if (payload.Length != Suites.Version.RegisterLength) + return new ProtocolTarget(null, prereleaseRequested); + + var protocolVersion = Suites.Version.GetPayload(reply).ProtocolVersion; + return protocolVersion.Major == 0 + ? new ProtocolTarget(null, prereleaseRequested) + : new ProtocolTarget(protocolVersion, prereleaseRequested); + } + catch (Exception) when (!cancellationToken.IsCancellationRequested) + { + return new ProtocolTarget(null, prereleaseRequested); + } + } +} diff --git a/src/Harp.Toolkit/Verify/Report.cs b/src/Harp.Toolkit/Verify/Report.cs index 1f7f1ee..eb42cd4 100644 --- a/src/Harp.Toolkit/Verify/Report.cs +++ b/src/Harp.Toolkit/Verify/Report.cs @@ -5,6 +5,9 @@ public class Report public string DeviceName { get; set; } = "Unknown Device"; public DateTime RunDate { get; set; } = DateTime.Now; public bool IncludePrerelease { get; set; } - public int PrereleaseTestCount { get; set; } + public string ProtocolNotice { get; set; } = string.Empty; + public string DeclaredProtocolVersion { get; set; } = string.Empty; + public string CheckedProtocolVersion { get; set; } = string.Empty; + public string RegisterSetVersion { get; set; } = string.Empty; public List Suites { get; set; } = new(); } diff --git a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml index 419d4c0..91ac5d0 100644 --- a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml +++ b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml @@ -32,20 +32,24 @@ - @if (Model.PrereleaseTestCount > 0) +
+
+
+
Protocol version declared
+
@Model.DeclaredProtocolVersion
+
Checked against
+
@Model.CheckedProtocolVersion
+
Register set
+
Harp.Generators @Model.RegisterSetVersion
+
+
+
+ + @if (Model.ProtocolNotice.Length > 0) { - @if (Model.IncludePrerelease) - { - - } - else - { - - } + } @foreach (var suite in Model.Suites) diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index 1675d7c..99fd6d0 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -100,25 +100,29 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve AnsiConsole.MarkupLine($" [green]Done![/] ({deviceMetadata.Registers.Count} registers)"); } - var runner = new CoreRunner(prerelease, clockOptions, deviceMetadata, deviceRawYaml); - if (runner.PrereleaseTestCount > 0) + using var connection = await VerifyConnection.OpenAsync(portName, cancellationToken); + var target = await ProtocolTarget.ResolveAsync(connection, prerelease, cancellationToken); + var runner = new CoreRunner(target.IncludePrerelease, clockOptions, deviceMetadata, deviceRawYaml); + var notice = GetProtocolNotice(target, runner.PrereleaseTestCount); + + AnsiConsole.MarkupLine(DescribeProtocolSelection(target)); + if (notice.Length > 0) { - if (prerelease) - AnsiConsole.MarkupLine($"Including [bold]{runner.PrereleaseTestCount}[/] prerelease checks."); - else - AnsiConsole.MarkupLine($"[yellow]{runner.PrereleaseTestCount} prerelease checks were not run. Rerun with --prerelease to include them.[/]"); + var style = target.IncludePrerelease ? "grey" : "yellow"; + AnsiConsole.MarkupLine($"[{style}]{Markup.Escape(notice)}[/]"); } var report = new Report { DeviceName = $"Harp Device ({portName})", RunDate = DateTime.Now, - IncludePrerelease = prerelease, - PrereleaseTestCount = runner.PrereleaseTestCount + IncludePrerelease = target.IncludePrerelease, + ProtocolNotice = notice, + DeclaredProtocolVersion = GetDeclaredVersion(target), + CheckedProtocolVersion = GetCheckedVersion(target), + RegisterSetVersion = CoreSchema.Version }; - using var connection = await VerifyConnection.OpenAsync(portName, cancellationToken); - int currentTest = 0; await foreach (var (suite, result) in runner.RunAllAsync(connection, cancellationToken, (suite, testName, testDesc) => { @@ -206,6 +210,63 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve } } + static string GetDeclaredVersion(ProtocolTarget target) + { + return target.DeclaredVersion.HasValue + ? target.DeclaredVersion.GetValueOrDefault().ToString() + : "not declared"; + } + + static string DescribeProtocolSelection(ProtocolTarget target) + { + if (target.IncludePrerelease) + { + return $"Checking against protocol version [bold]{GetDeclaredVersion(target)}[/], " + + "which is not yet ratified."; + } + + return $"Protocol version [bold]{GetDeclaredVersion(target)}[/], " + + $"checking against [bold]{GetCheckedVersion(target)}[/]."; + } + + static string GetCheckedVersion(ProtocolTarget target) + { + if (target.IncludePrerelease) + return $"{GetDeclaredVersion(target)}, which is not yet ratified"; + + return target.Scope == ProtocolScope.V2 + ? "v1, since v2 is not yet ratified" + : "v1"; + } + + static string GetProtocolNotice(ProtocolTarget target, int count) + { + if (target.Scope == ProtocolScope.Unsupported) + { + return $"This device declares protocol {GetDeclaredVersion(target)}, which this toolkit " + + "does not cover. The results below are against the v1 baseline only."; + } + + if (target.Scope == ProtocolScope.V1) + { + if (target.DeclaredVersion.HasValue) + return $"This device declares protocol {GetDeclaredVersion(target)}, so only the v1 baseline applies."; + + var recommendation = "Updating to a firmware that implements R_VERSION would let it be " + + "verified against the current protocol."; + return target.PrereleaseRequested + ? $"This device declares no protocol version, so --prerelease had no effect. {recommendation}" + : $"This device declares no protocol version. {recommendation}"; + } + + if (count == 0) + return string.Empty; + + return target.PrereleaseRequested + ? $"Including {count} prerelease checks, which this device declares support for." + : $"{count} prerelease checks were not run. Rerun with --prerelease to include them."; + } + static string GetResultMarkup(IResult result) { return result.Status switch From df855c2aee1313a2d3405a584a2d497ccf4840d1 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 8 Sep 2026 10:49:25 +0100 Subject: [PATCH 32/41] Fix test mismatches with current protocol text The serial number test asserted a derivation from R_UID that was added after the v1.13.0 tag and removed again in protocol 225, so it failed on every conformant device. It is replaced by a readability test, which the deprecated register rules still require. The timestamp offset writability test asserted the opposite of every version of the specification, where the register is marked writable at the tag and on main, and is removed. The two operation control heartbeat tests are marked prerelease, since they write a bit and await events from a register the tagged specification does not define. A device without R_VERSION now passes the whole default run, which is 28 tests. --- .../Suites/CoreRegisters/R_OPERATION_CTRL.cs | 4 ++-- .../Suites/CoreRegisters/R_SERIAL_NUMBER.cs | 21 ++++++------------- .../CoreRegisters/R_TIMESTAMP_OFFSET.cs | 16 +------------- 3 files changed, 9 insertions(+), 32 deletions(-) diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs index 1e4b3e1..87ce1e2 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_OPERATION_CTRL.cs @@ -50,7 +50,7 @@ public async Task VisualEnWritable(VerifyConnection device) return await TestOptionalBitAsync(device, "VisualEn", 0x20); } - [HarpTest(Description = "Validates that enabling HEARTBEAT_EN causes the device to emit R_HEARTBEAT events.")] + [HarpTest(Description = "Validates that enabling HEARTBEAT_EN causes the device to emit R_HEARTBEAT events.", Prerelease = true)] public async Task HeartbeatEnEmitsEvents(VerifyConnection device) { byte? originalOpCtrl = null; @@ -80,7 +80,7 @@ public async Task HeartbeatEnEmitsEvents(VerifyConnection device) } } - [HarpTest(Description = "Validates that HEARTBEAT_EN (bit 2) takes precedence over ALIVE_EN (bit 7): when both are set, R_HEARTBEAT events are emitted and R_TIMESTAMP_SECOND events are not.")] + [HarpTest(Description = "Validates that HEARTBEAT_EN (bit 2) takes precedence over ALIVE_EN (bit 7): when both are set, R_HEARTBEAT events are emitted and R_TIMESTAMP_SECOND events are not.", Prerelease = true)] public async Task HeartbeatEnPrecedenceOverAliveEn(VerifyConnection device) { byte? originalOpCtrl = null; diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs index b6256b2..dab5c5c 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_SERIAL_NUMBER.cs @@ -1,23 +1,14 @@ -namespace Harp.Toolkit.Verify.Suites; +using Bonsai.Harp; + +namespace Harp.Toolkit.Verify.Suites; internal class R_SERIAL_NUMBER : Suite { public override string Description => "Serial Number Register Tests"; - [HarpTest(Description = "Validates that SerialNumber matches the first two bytes of R_UID.", Prerelease = true)] - public async Task AssertConsistentWithUid(VerifyConnection device) + [HarpTest(Description = "Validates that SerialNumber register is readable.")] + public async Task IsReadable(VerifyConnection device) { - var uidValue = await device.ReadByteArrayAsync(R_UID.Address); - if (uidValue.Length < 2) - throw new ArgumentException($"Expected UID register contents to be at least 2 bytes. Got {uidValue.Length}."); - var twoFirstBytes = BitConverter.ToInt16(uidValue, 0); - - var serialNumberValue = await device.ReadSerialNumberAsync(); - - return new AssertionResult( - twoFirstBytes == serialNumberValue, - x => x ? - "SerialNumber register contents are consistent with UID register." : - $"SerialNumber register content (0x{serialNumberValue:X4}) does not match the first two bytes of UID register (0x{twoFirstBytes:X4})."); + return await RegisterHelpers.AssertReadableAsync(a => device.ReadUInt16Async(a), SerialNumber.Address, "SerialNumber"); } } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs index cf86d27..af2213d 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_OFFSET.cs @@ -1,6 +1,4 @@ - -using Bonsai.Harp; -namespace Harp.Toolkit.Verify.Suites; +namespace Harp.Toolkit.Verify.Suites; internal class R_TIMESTAMP_OFFSET : Suite { @@ -17,16 +15,4 @@ public async Task AssertReturnsZero(VerifyConnection device) "TimestampOffset register correctly returned 0x00." : $"TimestampOffset register returned a non-zero value (0x{value:X2})."); } - - [HarpTest(Description = "Validates the deprecated register TimestampOffset is NOT writable.", Prerelease = true)] - public async Task IsNotWritable(VerifyConnection device) - { - var req = HarpMessage.FromByte(Address, MessageType.Write, 0x00); - var rejected = await RegisterHelpers.IsWriteRejectedAsync(device, req); - return new AssertionResult( - rejected, - x => x ? - "Device correctly reported an error when trying to write to TimestampOffset register." : - "Timestamp Offset register is deprecated and MUST NOT allow writes."); - } } From 50bed2576b21e04d9b8e47a579adaf1b36b24bec Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 8 Sep 2026 17:06:10 +0100 Subject: [PATCH 33/41] Rename the device metadata option to --metadata The verify option naming the device interface file was --device-yml, which named a serialization format rather than the content, and diverged from the word the toolkit already uses for that file in DeviceMetadata, ReadDeviceMetadata and MetadataPathArgument. --- .../Verify/Suites/DeviceInterfaceSuite.cs | 4 ++-- src/Harp.Toolkit/Verify/VerifyCommand.cs | 22 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs b/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs index 10ce7f3..99fcced 100644 --- a/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/DeviceInterfaceSuite.cs @@ -64,7 +64,7 @@ public DeviceInterfaceSuite(DeviceMetadata? metadata, string? rawYaml) public Task GenerateAndCompileInterface(VerifyConnection device) { IResult result = metadata is null - ? new Result(false, Status.Skipped, "No device.yml provided (--device-yml).") + ? new Result(false, Status.Skipped, "No device metadata provided (--metadata).") : registerMap is not null ? new AssertionResult(true, $"Generated and compiled the interface with {registerMap.Count} registers.") : new ErrorResult(compileError!); @@ -75,7 +75,7 @@ public Task GenerateAndCompileInterface(VerifyConnection device) public async Task DeviceIdentity(VerifyConnection device) { if (metadata is null) - return new Result(false, Status.Skipped, "No device.yml provided (--device-yml)."); + return new Result(false, Status.Skipped, "No device metadata provided (--metadata)."); var mismatches = new List(); diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index 99fd6d0..bcc9352 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -53,12 +53,12 @@ public VerifyCommand() result.AddError("The number of clock samples must be greater than zero."); }); - Option deviceYmlOption = new("--device-yml") + Option metadataOption = new("--metadata") { - Description = "Path to the device's device.yml. Enables validation of the generated C# interface against a live read of every declared register, and cross-checks WhoAmI/firmware/hardware versions.", + Description = "The path to the file describing the device registers. Enables validation of the generated interface against a live read of every declared register, and cross-checks the WhoAmI, firmware and hardware versions.", Required = false, }; - OptionValidation.AcceptExistingOnly(deviceYmlOption); + OptionValidation.AcceptExistingOnly(metadataOption); Options.Add(portNameOption); Options.Add(fileOption); @@ -67,7 +67,7 @@ public VerifyCommand() Options.Add(clockPortOption); Options.Add(ppsEventOption); Options.Add(clockSamplesOption); - Options.Add(deviceYmlOption); + Options.Add(metadataOption); SetAction(parsedResult => { string portName = parsedResult.GetRequiredValue(portNameOption); @@ -79,12 +79,12 @@ public VerifyCommand() ClockPort: clockPort, PpsEvent: parsedResult.GetValue(ppsEventOption), ClockSamples: parsedResult.GetValue(clockSamplesOption)); - FileInfo? deviceYml = parsedResult.GetValue(deviceYmlOption); - return RunVerification(portName, reportFile, verbose, prerelease, clockOptions, deviceYml, CancellationToken.None); + FileInfo? metadataPath = parsedResult.GetValue(metadataOption); + return RunVerification(portName, reportFile, verbose, prerelease, clockOptions, metadataPath, CancellationToken.None); }); } - static async Task RunVerification(string portName, FileInfo? reportFile, bool verbose, bool prerelease, ClockTestOptions? clockOptions, FileInfo? deviceYml, CancellationToken cancellationToken) + static async Task RunVerification(string portName, FileInfo? reportFile, bool verbose, bool prerelease, ClockTestOptions? clockOptions, FileInfo? metadataPath, CancellationToken cancellationToken) { AnsiConsole.MarkupLine($"Running tests on [bold]{portName}[/]..."); if (clockOptions is not null) @@ -92,11 +92,11 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve DeviceMetadata? deviceMetadata = null; string? deviceRawYaml = null; - if (deviceYml is not null) + if (metadataPath is not null) { - AnsiConsole.Markup($"Loading device metadata from [bold]{deviceYml.FullName}[/]..."); - deviceMetadata = GeneratorHelper.ReadDeviceMetadata(deviceYml.FullName); - deviceRawYaml = await File.ReadAllTextAsync(deviceYml.FullName, cancellationToken); + AnsiConsole.Markup($"Loading device metadata from [bold]{metadataPath.FullName}[/]..."); + deviceMetadata = GeneratorHelper.ReadDeviceMetadata(metadataPath.FullName); + deviceRawYaml = await File.ReadAllTextAsync(metadataPath.FullName, cancellationToken); AnsiConsole.MarkupLine($" [green]Done![/] ({deviceMetadata.Registers.Count} registers)"); } From a0ca875ef71ab2c02c1788923f4c9721a4b44160 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 9 Sep 2026 13:57:54 +0100 Subject: [PATCH 34/41] Connect in standby and bound the identity read The shared connection now requests standby mode with register dumps disabled, so the board no longer streams application events during a run. The identity read gains a two-second deadline per attempt. Each phase now starts its own retry budget rather than sharing one, so the connection always gets its full retry window. Both phases also retry transport errors and timeouts. The run no longer hangs. Stale bytes on a reopened port still produce an occasional parse warning, but the parser resynchronizes and the run completes. --- src/Harp.Toolkit/Verify/VerifyConnection.cs | 48 ++++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/src/Harp.Toolkit/Verify/VerifyConnection.cs b/src/Harp.Toolkit/Verify/VerifyConnection.cs index 5cd021d..c448ed3 100644 --- a/src/Harp.Toolkit/Verify/VerifyConnection.cs +++ b/src/Harp.Toolkit/Verify/VerifyConnection.cs @@ -1,4 +1,5 @@ -using System.Reactive.Linq; +using System.Diagnostics; +using System.Reactive.Linq; using System.Reactive.Subjects; using Bonsai.Harp; @@ -12,7 +13,7 @@ public sealed class VerifyConnection : IDisposable const int ConnectDelayMilliseconds = 200; const int PortReleaseDelayMilliseconds = 300; const int OpenTimeoutMilliseconds = 10000; - const int OpenRetryDelayMilliseconds = 250; + const int IdentityReadTimeoutMilliseconds = 2000; const int ReadyAttempts = 5; const int ReadyTimeoutMilliseconds = 1000; @@ -22,7 +23,13 @@ public sealed class VerifyConnection : IDisposable VerifyConnection(string portName, int whoAmI) { - var device = new Bonsai.Harp.Device(whoAmI) { PortName = portName, IgnoreErrors = true }; + var device = new Bonsai.Harp.Device(whoAmI) + { + PortName = portName, + IgnoreErrors = true, + OperationMode = OperationMode.Standby, + DumpRegisters = false, + }; messages = device.Generate(requests).Publish(); subscription = messages.Connect(); } @@ -34,10 +41,10 @@ public sealed class VerifyConnection : IDisposable /// public static async Task OpenAsync(string portName, CancellationToken cancellationToken = default) { - var deadline = Environment.TickCount64 + OpenTimeoutMilliseconds; - var whoAmI = await ReadIdentityAsync(portName, deadline, cancellationToken); + var whoAmI = await ReadIdentityAsync(portName, cancellationToken); await Task.Delay(PortReleaseDelayMilliseconds, cancellationToken); + var retryStart = Stopwatch.GetTimestamp(); while (true) { var connection = new VerifyConnection(portName, whoAmI); @@ -46,10 +53,13 @@ public static async Task OpenAsync(string portName, Cancellati await connection.WaitUntilReadyAsync(cancellationToken); return connection; } - catch (UnauthorizedAccessException) when (Environment.TickCount64 < deadline && !cancellationToken.IsCancellationRequested) + catch (Exception ex) when ( + IsRetryableOpenFailure(ex) && + IsWithinRetryBudget(retryStart) && + !cancellationToken.IsCancellationRequested) { connection.Dispose(); - await Task.Delay(OpenRetryDelayMilliseconds, cancellationToken); + await Task.Delay(PortReleaseDelayMilliseconds, cancellationToken); } catch { @@ -59,22 +69,38 @@ public static async Task OpenAsync(string portName, Cancellati } } - static async Task ReadIdentityAsync(string portName, long deadline, CancellationToken cancellationToken) + static async Task ReadIdentityAsync(string portName, CancellationToken cancellationToken) { + var retryStart = Stopwatch.GetTimestamp(); while (true) { + using var readTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + readTimeout.CancelAfter(IdentityReadTimeoutMilliseconds); try { using var probe = new AsyncDevice(portName); - return await probe.ReadWhoAmIAsync(cancellationToken); + return await probe.ReadWhoAmIAsync(readTimeout.Token); } - catch (UnauthorizedAccessException) when (Environment.TickCount64 < deadline && !cancellationToken.IsCancellationRequested) + catch (Exception ex) when ( + (IsRetryableOpenFailure(ex) || ex is OperationCanceledException) && + IsWithinRetryBudget(retryStart) && + !cancellationToken.IsCancellationRequested) { - await Task.Delay(OpenRetryDelayMilliseconds, cancellationToken); + await Task.Delay(PortReleaseDelayMilliseconds, cancellationToken); } } } + static bool IsRetryableOpenFailure(Exception ex) + { + return ex is UnauthorizedAccessException || ex is IOException || ex is TimeoutException; + } + + static bool IsWithinRetryBudget(long retryStart) + { + return Stopwatch.GetElapsedTime(retryStart).TotalMilliseconds < OpenTimeoutMilliseconds; + } + /// /// Every message received from the device, before any reply correlation. /// From da6ee5ca84eff9b67c99004429a3a84a30f675c1 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 9 Sep 2026 16:40:17 +0100 Subject: [PATCH 35/41] Bound the timestamp write for a running clock Writing the register sets the integer seconds and leaves the sub-second counter running, so the clock can cross a second boundary before it is read. The bound now allows one full crossing and is directional, so a clock reading below the written value fails instead of passing. The failure message reports the reply timestamp against the expected range. --- .../Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs index 89683fd..d6f8c88 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs @@ -10,13 +10,18 @@ internal class R_TIMESTAMP_SECOND : Suite public async Task IsWritable(VerifyConnection device) { const uint setSeconds = 42; + const double maximumElapsedSeconds = 2.0; await device.WriteTimestampSecondsAsync(setSeconds); await Task.Delay(1); HarpMessage response = await device.CommandAsync(TimestampSeconds.FromPayload(MessageType.Read, default)); double readSeconds = response.GetTimestamp(); + double elapsedSeconds = readSeconds - setSeconds; return new AssertionResult( - Math.Abs(readSeconds - setSeconds) < 1.0, - (success) => success ? "TimestampSeconds register is writable and updates as expected." : $"TimestampSeconds register is not writable. Expected value: {setSeconds}, read value: {readSeconds}."); + elapsedSeconds >= 0 && elapsedSeconds < maximumElapsedSeconds, + (success) => success + ? "TimestampSeconds register is writable and updates as expected." + : $"Wrote {setSeconds} to TimestampSeconds and the reply timestamp was {readSeconds:F6}, " + + $"outside the expected range of {setSeconds} to {setSeconds + maximumElapsedSeconds}."); } [HarpTest(Description = "Validates that TimestampSeconds register is readable.")] From 4e4efc7ef6ba3c43ba338df6221f30478aeb12ac Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 9 Sep 2026 21:34:43 +0100 Subject: [PATCH 36/41] Identify the device in the console and report The device identity is read once before the suite runs, each read bounded by its own 2000 ms deadline, so an unanswered read shows as not reported rather than stalling the run. The console prints one identity line and the report header carries WhoAmI, serial port, hardware version and firmware version. The report is now titled with the device name, with a Harp conformance report subtitle in place of the toolkit badge. --- src/Harp.Toolkit/Verify/DeviceIdentity.cs | 9 ++++ src/Harp.Toolkit/Verify/Report.cs | 4 ++ src/Harp.Toolkit/Verify/ReportTemplate.cshtml | 19 ++++---- src/Harp.Toolkit/Verify/VerifyCommand.cs | 16 ++++++- src/Harp.Toolkit/Verify/VerifyConnection.cs | 45 ++++++++++++++++++- 5 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 src/Harp.Toolkit/Verify/DeviceIdentity.cs diff --git a/src/Harp.Toolkit/Verify/DeviceIdentity.cs b/src/Harp.Toolkit/Verify/DeviceIdentity.cs new file mode 100644 index 0000000..083d3c1 --- /dev/null +++ b/src/Harp.Toolkit/Verify/DeviceIdentity.cs @@ -0,0 +1,9 @@ +using Bonsai.Harp; + +namespace Harp.Toolkit.Verify; + +internal readonly record struct DeviceIdentity( + int WhoAmI, + string? Name, + HarpVersion? HardwareVersion, + HarpVersion? FirmwareVersion); diff --git a/src/Harp.Toolkit/Verify/Report.cs b/src/Harp.Toolkit/Verify/Report.cs index eb42cd4..0e68bf4 100644 --- a/src/Harp.Toolkit/Verify/Report.cs +++ b/src/Harp.Toolkit/Verify/Report.cs @@ -3,6 +3,10 @@ public class Report { public string DeviceName { get; set; } = "Unknown Device"; + public string PortName { get; set; } = string.Empty; + public string WhoAmI { get; set; } = string.Empty; + public string HardwareVersion { get; set; } = string.Empty; + public string FirmwareVersion { get; set; } = string.Empty; public DateTime RunDate { get; set; } = DateTime.Now; public bool IncludePrerelease { get; set; } public string ProtocolNotice { get; set; } = string.Empty; diff --git a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml index 91ac5d0..22df973 100644 --- a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml +++ b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml @@ -22,19 +22,22 @@
-
-
-

@Model.DeviceName

-

Test Execution Report • @Model.RunDate.ToString("MMMM dd, yyyy HH:mm:ss")

-
-
- Harp Toolkit Test -
+
+

@Model.DeviceName

+

Harp conformance report • @Model.RunDate.ToString("MMMM dd, yyyy HH:mm:ss")

+
WhoAmI
+
@Model.WhoAmI
+
Serial port
+
@Model.PortName
+
Hardware version
+
@Model.HardwareVersion
+
Firmware version
+
@Model.FirmwareVersion
Protocol version declared
@Model.DeclaredProtocolVersion
Checked against
diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index bcc9352..f1eb1e6 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -101,10 +101,12 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve } using var connection = await VerifyConnection.OpenAsync(portName, cancellationToken); + var identity = await connection.ReadDeviceIdentityAsync(cancellationToken); var target = await ProtocolTarget.ResolveAsync(connection, prerelease, cancellationToken); var runner = new CoreRunner(target.IncludePrerelease, clockOptions, deviceMetadata, deviceRawYaml); var notice = GetProtocolNotice(target, runner.PrereleaseTestCount); + AnsiConsole.MarkupLine(DescribeDeviceIdentity(identity, portName)); AnsiConsole.MarkupLine(DescribeProtocolSelection(target)); if (notice.Length > 0) { @@ -114,7 +116,11 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve var report = new Report { - DeviceName = $"Harp Device ({portName})", + DeviceName = identity.Name is { Length: > 0 } name ? name : "Harp Device", + PortName = portName, + WhoAmI = identity.WhoAmI.ToString(), + HardwareVersion = identity.HardwareVersion?.ToString() ?? "not reported", + FirmwareVersion = identity.FirmwareVersion?.ToString() ?? "not reported", RunDate = DateTime.Now, IncludePrerelease = target.IncludePrerelease, ProtocolNotice = notice, @@ -217,6 +223,14 @@ static string GetDeclaredVersion(ProtocolTarget target) : "not declared"; } + static string DescribeDeviceIdentity(DeviceIdentity identity, string portName) + { + var name = identity.Name is { Length: > 0 } deviceName ? deviceName : "unnamed device"; + return $"Device [bold]{Markup.Escape(name)}[/] on {portName}, WhoAmI [bold]{identity.WhoAmI}[/], " + + $"hardware {identity.HardwareVersion?.ToString() ?? "not reported"}, " + + $"firmware {identity.FirmwareVersion?.ToString() ?? "not reported"}."; + } + static string DescribeProtocolSelection(ProtocolTarget target) { if (target.IncludePrerelease) diff --git a/src/Harp.Toolkit/Verify/VerifyConnection.cs b/src/Harp.Toolkit/Verify/VerifyConnection.cs index c448ed3..0150bc7 100644 --- a/src/Harp.Toolkit/Verify/VerifyConnection.cs +++ b/src/Harp.Toolkit/Verify/VerifyConnection.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Reactive.Linq; using System.Reactive.Subjects; +using System.Text; using Bonsai.Harp; namespace Harp.Toolkit.Verify; @@ -23,6 +24,7 @@ public sealed class VerifyConnection : IDisposable VerifyConnection(string portName, int whoAmI) { + WhoAmI = whoAmI; var device = new Bonsai.Harp.Device(whoAmI) { PortName = portName, @@ -101,6 +103,11 @@ static bool IsWithinRetryBudget(long retryStart) return Stopwatch.GetElapsedTime(retryStart).TotalMilliseconds < OpenTimeoutMilliseconds; } + /// + /// The device identifier validated when the connection was opened. + /// + public int WhoAmI { get; } + /// /// Every message received from the device, before any reply correlation. /// @@ -185,7 +192,14 @@ public async Task ReadUInt32Async(int address, CancellationToken cancellat public async Task ReadWhoAmIAsync(CancellationToken cancellationToken = default) { - return await ReadUInt16Async(WhoAmI.Address, cancellationToken); + return await ReadUInt16Async(Bonsai.Harp.WhoAmI.Address, cancellationToken); + } + + public async Task ReadDeviceNameAsync(CancellationToken cancellationToken = default) + { + var payload = await ReadByteArrayAsync(DeviceName.Address, cancellationToken); + var terminator = Array.IndexOf(payload, (byte)0); + return Encoding.ASCII.GetString(payload, 0, terminator < 0 ? payload.Length : terminator); } public async Task ReadAssemblyVersionAsync(CancellationToken cancellationToken = default) @@ -222,6 +236,35 @@ public async Task WriteTimestampSecondsAsync(uint seconds, CancellationToken can await CommandAsync(HarpCommand.WriteUInt32(TimestampSeconds.Address, seconds), cancellationToken); } + /// + /// Reads the identity registers reported in the run header, leaving any register the device + /// does not answer within the read timeout unreported rather than failing the run. + /// + internal async Task ReadDeviceIdentityAsync(CancellationToken cancellationToken = default) + { + var name = await TryReadAsync(ReadDeviceNameAsync, cancellationToken); + var hardwareVersion = await TryReadAsync(ReadHardwareVersionAsync, cancellationToken); + var firmwareVersion = await TryReadAsync(ReadFirmwareVersionAsync, cancellationToken); + return new DeviceIdentity(WhoAmI, name, hardwareVersion, firmwareVersion); + } + + static async Task TryReadAsync( + Func> read, + CancellationToken cancellationToken) + where T : class + { + using var readTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + readTimeout.CancelAfter(IdentityReadTimeoutMilliseconds); + try + { + return await read(readTimeout.Token); + } + catch (Exception) when (!cancellationToken.IsCancellationRequested) + { + return null; + } + } + async Task WaitUntilReadyAsync(CancellationToken cancellationToken) { await Task.Delay(ConnectDelayMilliseconds, cancellationToken); From 2b73b13260542532e003c54aec1a30c4b7ef71aa Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 9 Sep 2026 21:52:44 +0100 Subject: [PATCH 37/41] Bound device requests with a reply timeout Requests sent through CommandAsync now fail with a TimeoutException when the device does not reply within 2000 ms, so an unanswered request fails only that test instead of stalling the whole run. The reply timeout, the identity reads and the connection probe share one constant. The protocol version read moves onto the connection as ReadProtocolVersionAsync, taking the payload length check and the all-zero normalization with it, which leaves ProtocolTarget a plain record struct over its declared version and the prerelease flag. --- src/Harp.Toolkit/Verify/ProtocolTarget.cs | 31 +-------------- src/Harp.Toolkit/Verify/VerifyCommand.cs | 3 +- src/Harp.Toolkit/Verify/VerifyConnection.cs | 43 +++++++++++++++++---- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/Harp.Toolkit/Verify/ProtocolTarget.cs b/src/Harp.Toolkit/Verify/ProtocolTarget.cs index 6ab3ae7..621da97 100644 --- a/src/Harp.Toolkit/Verify/ProtocolTarget.cs +++ b/src/Harp.Toolkit/Verify/ProtocolTarget.cs @@ -1,5 +1,4 @@ -using Bonsai.Harp; -using Harp.Toolkit.Verify.Suites; +using Harp.Toolkit.Verify.Suites; namespace Harp.Toolkit.Verify; @@ -13,7 +12,6 @@ internal enum ProtocolScope internal readonly record struct ProtocolTarget(SemanticVersion? DeclaredVersion, bool PrereleaseRequested) { const int PrereleaseMajorVersion = 2; - const int ReadTimeoutMilliseconds = 2000; public ProtocolScope Scope { @@ -28,31 +26,4 @@ public ProtocolScope Scope } public bool IncludePrerelease => Scope == ProtocolScope.V2 && PrereleaseRequested; - - public static async Task ResolveAsync( - VerifyConnection connection, - bool prereleaseRequested, - CancellationToken cancellationToken = default) - { - using var readTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - readTimeout.CancelAfter(ReadTimeoutMilliseconds); - try - { - var reply = await connection.CommandAsync( - HarpCommand.ReadByte(Suites.Version.Address), - readTimeout.Token); - var payload = reply.GetPayloadArray(); - if (payload.Length != Suites.Version.RegisterLength) - return new ProtocolTarget(null, prereleaseRequested); - - var protocolVersion = Suites.Version.GetPayload(reply).ProtocolVersion; - return protocolVersion.Major == 0 - ? new ProtocolTarget(null, prereleaseRequested) - : new ProtocolTarget(protocolVersion, prereleaseRequested); - } - catch (Exception) when (!cancellationToken.IsCancellationRequested) - { - return new ProtocolTarget(null, prereleaseRequested); - } - } } diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index f1eb1e6..70ee3a4 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -102,7 +102,8 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve using var connection = await VerifyConnection.OpenAsync(portName, cancellationToken); var identity = await connection.ReadDeviceIdentityAsync(cancellationToken); - var target = await ProtocolTarget.ResolveAsync(connection, prerelease, cancellationToken); + var declaredVersion = await connection.ReadProtocolVersionAsync(cancellationToken); + var target = new ProtocolTarget(declaredVersion, prerelease); var runner = new CoreRunner(target.IncludePrerelease, clockOptions, deviceMetadata, deviceRawYaml); var notice = GetProtocolNotice(target, runner.PrereleaseTestCount); diff --git a/src/Harp.Toolkit/Verify/VerifyConnection.cs b/src/Harp.Toolkit/Verify/VerifyConnection.cs index 0150bc7..0f4d75f 100644 --- a/src/Harp.Toolkit/Verify/VerifyConnection.cs +++ b/src/Harp.Toolkit/Verify/VerifyConnection.cs @@ -14,7 +14,7 @@ public sealed class VerifyConnection : IDisposable const int ConnectDelayMilliseconds = 200; const int PortReleaseDelayMilliseconds = 300; const int OpenTimeoutMilliseconds = 10000; - const int IdentityReadTimeoutMilliseconds = 2000; + const int ReadTimeoutMilliseconds = 2000; const int ReadyAttempts = 5; const int ReadyTimeoutMilliseconds = 1000; @@ -77,7 +77,7 @@ static async Task ReadIdentityAsync(string portName, CancellationToken canc while (true) { using var readTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - readTimeout.CancelAfter(IdentityReadTimeoutMilliseconds); + readTimeout.CancelAfter(ReadTimeoutMilliseconds); try { using var probe = new AsyncDevice(portName); @@ -118,8 +118,14 @@ static bool IsWithinRetryBudget(long retryStart) ///
public void Write(HarpMessage message) => requests.OnNext(message); + /// + /// Sends the specified request and awaits the matching reply, failing with a + /// if the device does not answer in time. + /// public async Task CommandAsync(HarpMessage command, CancellationToken cancellationToken = default) { + using var replyTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + replyTimeout.CancelAfter(ReadTimeoutMilliseconds); var reply = messages.FirstAsync(message => { var match = message.IsMatch(command.Address, command.MessageType); @@ -129,10 +135,19 @@ public async Task CommandAsync(HarpMessage command, CancellationTok } return match; - }).RunAsync(cancellationToken); + }).RunAsync(replyTimeout.Token); Write(command); - return await reply; + try + { + return await reply; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException( + $"The device did not reply to a {command.MessageType} request at address " + + $"{command.Address} within {ReadTimeoutMilliseconds} ms."); + } } /// @@ -248,16 +263,30 @@ internal async Task ReadDeviceIdentityAsync(CancellationToken ca return new DeviceIdentity(WhoAmI, name, hardwareVersion, firmwareVersion); } + /// + /// Reads the protocol version the device declares, leaving it undeclared when the version + /// register is unreadable, carries an unexpected length, or reports all zeros. + /// + internal async Task ReadProtocolVersionAsync(CancellationToken cancellationToken = default) + { + var reply = await TryReadAsync( + token => CommandAsync(HarpCommand.ReadByte(Suites.Version.Address), token), + cancellationToken); + if (reply is null || reply.GetPayloadArray().Length != Suites.Version.RegisterLength) + return null; + + var protocolVersion = Suites.Version.GetPayload(reply).ProtocolVersion; + return protocolVersion.Major == 0 ? null : protocolVersion; + } + static async Task TryReadAsync( Func> read, CancellationToken cancellationToken) where T : class { - using var readTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - readTimeout.CancelAfter(IdentityReadTimeoutMilliseconds); try { - return await read(readTimeout.Token); + return await read(cancellationToken); } catch (Exception) when (!cancellationToken.IsCancellationRequested) { From 611e5a46ee0cf1712657d90a3e2a0ee393211d35 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 9 Sep 2026 22:15:23 +0100 Subject: [PATCH 38/41] Cite the specification commit in the report The report header now names the harp-tech/protocol commit behind the checks, linked to the repository tree at that commit. --- src/Harp.Toolkit/Verify/ProtocolReference.cs | 22 +++++++++++++++++++ src/Harp.Toolkit/Verify/Report.cs | 2 ++ src/Harp.Toolkit/Verify/ReportTemplate.cshtml | 2 ++ src/Harp.Toolkit/Verify/VerifyCommand.cs | 2 ++ 4 files changed, 28 insertions(+) create mode 100644 src/Harp.Toolkit/Verify/ProtocolReference.cs diff --git a/src/Harp.Toolkit/Verify/ProtocolReference.cs b/src/Harp.Toolkit/Verify/ProtocolReference.cs new file mode 100644 index 0000000..da09d9b --- /dev/null +++ b/src/Harp.Toolkit/Verify/ProtocolReference.cs @@ -0,0 +1,22 @@ +namespace Harp.Toolkit.Verify; + +/// +/// Identifies the specification text encoded by the conformance checks. +/// +internal static class ProtocolReference +{ + /// + /// The harp-tech/protocol commit carrying that specification text. + /// + public const string Commit = "11b584bdd2eb45a8b55ebc540512864ff0667326"; + + /// + /// Gets the abbreviated commit, for display where a link carries the full reference. + /// + public static string ShortCommit => Commit[..7]; + + /// + /// Gets the address of the specification documents as they stood at that commit. + /// + public static string TreeUrl => $"https://github.com/harp-tech/protocol/tree/{Commit}"; +} diff --git a/src/Harp.Toolkit/Verify/Report.cs b/src/Harp.Toolkit/Verify/Report.cs index 0e68bf4..608d991 100644 --- a/src/Harp.Toolkit/Verify/Report.cs +++ b/src/Harp.Toolkit/Verify/Report.cs @@ -12,6 +12,8 @@ public class Report public string ProtocolNotice { get; set; } = string.Empty; public string DeclaredProtocolVersion { get; set; } = string.Empty; public string CheckedProtocolVersion { get; set; } = string.Empty; + public string ProtocolCommit { get; set; } = string.Empty; + public string ProtocolCommitUrl { get; set; } = string.Empty; public string RegisterSetVersion { get; set; } = string.Empty; public List Suites { get; set; } = new(); } diff --git a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml index 22df973..29943a1 100644 --- a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml +++ b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml @@ -42,6 +42,8 @@
@Model.DeclaredProtocolVersion
Checked against
@Model.CheckedProtocolVersion
+
Specification
+
@Model.ProtocolCommit
Register set
Harp.Generators @Model.RegisterSetVersion
diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index 70ee3a4..11592ba 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -127,6 +127,8 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bool ve ProtocolNotice = notice, DeclaredProtocolVersion = GetDeclaredVersion(target), CheckedProtocolVersion = GetCheckedVersion(target), + ProtocolCommit = ProtocolReference.ShortCommit, + ProtocolCommitUrl = ProtocolReference.TreeUrl, RegisterSetVersion = CoreSchema.Version }; From 6fe41e63b264c0b9d5817e3785d7973a659814c9 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 9 Sep 2026 22:33:25 +0100 Subject: [PATCH 39/41] Tighten the past value bound and fix suite wording Writing a past timestamp is now accepted only when the clock reads forward of the written value, where the previous absolute difference also accepted a clock behind it. The one second tolerance is unchanged, since this check compares integer seconds and a boundary crossing shows as exactly one. The round trip suite description no longer claims a set of tests that write registers, since it is one test that reads WhoAmI and reports latency statistics. The boot provenance failure message no longer cites the rule that a device without non-volatile memory must set BOOT_DEF. That rule postdates the stable baseline, where the check itself runs. --- .../Verify/Suites/CoreRegisters/R_RESET_DEV.cs | 2 +- .../Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs | 10 ++++++---- src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs index a1d01eb..d7c1e96 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_RESET_DEV.cs @@ -62,7 +62,7 @@ static string DescribeReadViolation(ResetFlags flags) if (provenance == BootProvenance) return "Both BOOT_DEF and BOOT_EE are set, so the reported boot provenance is contradictory."; if (provenance == 0) - return "Neither BOOT_DEF nor BOOT_EE is set, so no boot provenance is reported. A device without non-volatile memory must always set BOOT_DEF."; + return "Neither BOOT_DEF nor BOOT_EE is set, so no boot provenance is reported."; var commandBits = DescribeCommandBits(flags); if (commandBits.Length > 0) diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs index d6f8c88..4a6472f 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_TIMESTAMP_SECOND.cs @@ -54,6 +54,7 @@ public async Task IsMonotonic(VerifyConnection device) [HarpTest(Description = "Validates that writing a past timestamp value takes effect and can be read back.")] public async Task WritePastValueRoundTrip(VerifyConnection device) { + const long maximumElapsedSeconds = 1; var current = await device.ReadTimestampSecondsAsync(); var tPast = current >= 10 ? current - 10 : 0u; @@ -61,12 +62,13 @@ public async Task WritePastValueRoundTrip(VerifyConnection device) await Task.Delay(50); var readBack = await device.ReadTimestampSecondsAsync(); - bool withinBounds = Math.Abs((long)readBack - (long)tPast) <= 1; + var elapsedSeconds = (long)readBack - tPast; return new AssertionResult( - withinBounds, + elapsedSeconds >= 0 && elapsedSeconds <= maximumElapsedSeconds, x => x - ? $"WritePastValueRoundTrip: wrote {tPast}, read back {readBack} (within 1s tolerance)." - : $"WritePastValueRoundTrip: wrote {tPast}, read back {readBack} (difference {Math.Abs((long)readBack - (long)tPast)}s, expected <= 1)."); + ? $"Wrote {tPast} to TimestampSeconds and read it back as {readBack}." + : $"Wrote {tPast} to TimestampSeconds and read it back as {readBack}, " + + $"outside the expected range of {tPast} to {tPast + maximumElapsedSeconds}."); } } diff --git a/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs b/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs index ff98f27..45ca893 100644 --- a/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs +++ b/src/Harp.Toolkit/Verify/Suites/RoundTripTestSuite.cs @@ -5,7 +5,7 @@ namespace Harp.Toolkit.Verify.Suites; internal class RoundTripTestSuite : Suite { - public override string Description => "A bunch of tests to benchmark round trip read/writes."; + public override string Description => "Measures round trip latency statistics for a register read."; [HarpTest(Description = "Benchmarks the round trip time for a WhoAmI read command.")] public async Task BenchmarkRoundTrip(VerifyConnection device) From 3c906e6451309e72cce1cc3ef5f3e33dadf8b4b1 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 9 Sep 2026 22:51:30 +0100 Subject: [PATCH 40/41] Document the verify command A new article covers running a verification, how the specification revision is determined, the report structure and how to act on a reported failure, the synchronization clock checks and the declared interface checks, with every option documented. The readme gains a section pointing at it, and the article joins the table of contents after code generation. --- docs/README.md | 12 ++++ docs/articles/toc.yml | 3 +- docs/articles/verify.md | 124 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 docs/articles/verify.md diff --git a/docs/README.md b/docs/README.md index d60d975..82e97fd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -58,6 +58,18 @@ dotnet harp.toolkit generate interface python See [Code Generation](https://harp-tech.org/toolkit/articles/generate.html) for authoring device metadata, generating firmware, and the available options. +## Device Verification + +`harp.toolkit` can also check a device against the Harp specification, reporting where its behavior departs from the standard and writing the result as a shareable HTML report: + +```cmd +dotnet harp.toolkit verify --port COM4 --report report.html +``` + +Verification writes to device registers and assumes a freshly powered device, so avoid running it against a device that is part of a running experiment. + +See [Device Verification](https://harp-tech.org/toolkit/articles/verify.html) for the specification used to check the device, the report structure, and the available options. + ## Contributing Bug reports and contributions are welcome at [the GitHub repository](https://github.com/harp-tech/toolkit). diff --git a/docs/articles/toc.yml b/docs/articles/toc.yml index 1ce9c53..2571da1 100644 --- a/docs/articles/toc.yml +++ b/docs/articles/toc.yml @@ -1,3 +1,4 @@ - name: Introduction href: ../index.md -- href: generate.md \ No newline at end of file +- href: generate.md +- href: verify.md \ No newline at end of file diff --git a/docs/articles/verify.md b/docs/articles/verify.md new file mode 100644 index 0000000..3dbb867 --- /dev/null +++ b/docs/articles/verify.md @@ -0,0 +1,124 @@ +# Device Verification + +`harp.toolkit` can check a device against the Harp specification and report where its behavior departs from what the standard requires. The checks span all three specification documents, covering the core register set and its access rules, the reply behavior required by the binary protocol, and alignment on the synchronization clock. Results print to the console as the run proceeds, and can be written to a shareable HTML report. + +A verification result records how a device behaved against a stated revision of the specification, and it confers no compliance status. + +> [!Warning] +> Verification writes to device registers. Conformance cannot be established without exercising writes, read-only enforcement and event streams, so there is no read-only mode. Some checks leave the device clock and the operation control register in a changed state, and the run assumes a freshly powered device. Avoid verifying a device that is part of a running experiment. + +## Running a verification + +A verification needs only the serial port of the device. + +```ps1 +dotnet harp.toolkit verify --port COM3 +``` + +Every check reports as passed, failed or skipped. A check is skipped when it needs an option that was not supplied, and the message names the option. A device that stops answering fails the check that was waiting on it, after a fixed 2000 ms, so a silent register costs one result rather than stalling the rest of the run. + +#### Serial port +```ps1 +--port +``` + +Specifies the name of the serial port used to communicate with the device. This option is required. + +#### Detailed results +```ps1 +--verbose +``` + +Prints a detailed result for every check once the run finishes, including the statistics gathered by the measurements. Per-check progress is printed either way. + +## Specification version + +Harp devices do not all implement the same revision of the standard, so no single set of checks applies to every device. + +A device declares the revision it implements in `R_VERSION`. Where that register is absent, unreadable or reads all zeros, the device is held to v1, since a device that predates the register also predates the version field. Checks belonging to a revision outside that scope are neither run nor listed. A skipped result means a check that was in scope and did not run. The console and the report state how many checks were excluded. + +#### Include prerelease checks +```ps1 +--prerelease +``` + +Also runs the checks that encode specification text outside the stable baseline. Those checks apply only to a device declaring the matching major version, so supplying the option for a device that declares v1 changes nothing. A failure reported under this option may reflect text that is still being ratified, which makes it worth checking intent carefully against the specification. + +## Sharing a report + +Console output is not an artifact. A report captures one run as a single HTML file that can be attached to an issue or a release. + +```ps1 +dotnet harp.toolkit verify --port COM3 --report report.html +``` + +The report is titled with the device name and opens with a header describing the run. + +- **WhoAmI** is the device identity class, read from `R_WHO_AM_I`. +- **Serial port** is the port used to reach the device. +- **Hardware version** and **Firmware version** are read from the device at startup, and read as not reported for a device that does not answer them. +- **Protocol version declared** is what the device reports in `R_VERSION`, or that no version was declared. +- **Checked against** is the revision of the specification used to verify the device, together with the reason when that is narrower than what the device declared. +- **Specification** links to the specification documents as they stood at the commit behind the checks. +- **Register set** names the generator package supplying the core register metadata, which fully determines the register set the run expects. + +#### Report path +```ps1 +--report +``` + +Path of the HTML report written after the run. Without it the results are printed and not saved. + +### Acting on a reported failure + +The **Specification** link is what makes a disagreement decidable, so it is worth checking carefully before filing anything. Specification text moves between releases, and a check is written against one state of it. + +If the device matches the text at that commit and a check still fails, the check is wrong, and that belongs in the toolkit repository. If a check matches the text and the text itself is wrong, that belongs in the protocol repository. + +## Verifying the synchronization clock + +Supplying a second device as a clock reference enables the alignment checks. Both devices must be connected to the same synchronization clock bus. + +```ps1 +dotnet harp.toolkit verify --port COM3 --clock-port COM4 --pps-event 32 +``` + +#### Clock reference port +```ps1 +--clock-port +``` + +Serial port of the reference clock device. Supplying it enables the clock alignment checks. + +#### Tested device event register +```ps1 +--pps-event +``` + +Address of the register on the tested device that reports the incoming pulse from the reference clock device. Supplying it enables the pulse alignment check, which also requires a clock reference port. + +Note that the pulse is a physical output that only some devices produce, and it is distinct both from the synchronization signal on the clock bus and from the software heartbeat. A device reports the pulse through an application register of its own, which is why the address has to be supplied explicitly. + +#### Sample count +```ps1 +--clock-samples +``` + +Number of pulse event pairs to collect for the alignment check. The default is 5, and the value must be greater than zero. + +## Verifying the declared interface + +A device can also be checked against its own declared interface rather than only against the standard. Supplying the device metadata generates an interface from it, reads every declared register from the live device, and parses each reply with the generated parsers. The identity, firmware and hardware versions declared in the metadata are cross-checked against what the device reports. + +```ps1 +dotnet harp.toolkit verify --port COM3 --metadata device.yml +``` + +#### Device metadata +```ps1 +--metadata +``` + +Path of the file describing the device registers. The file must exist. + +Unlike code generation, this option has no default, so a `device.yml` located in the current directory does not automatically enable these checks. From b91665f5f08e0919cb47972f6e80f90bcf16913d Mon Sep 17 00:00:00 2001 From: glopesdev Date: Thu, 10 Sep 2026 23:31:16 +0100 Subject: [PATCH 41/41] Include prerelease checks for any declared version Prerelease checks now run whenever the flag is given, which previously also required the device to declare major 2. The version register gains a prerelease check that the declared major matches the version the checks target. Only the major is compared, since the prerelease text carries no settled version number. The checked-against value and the console selection line no longer derive from the declared version, which would misreport a device held to text it does not claim. The rerun hint now reaches devices declaring no version or a major outside coverage. --- src/Harp.Toolkit/Verify/ProtocolReference.cs | 5 ++ src/Harp.Toolkit/Verify/ProtocolTarget.cs | 8 ++-- .../Verify/Suites/CoreRegisters/R_VERSION.cs | 28 +++++++++++ src/Harp.Toolkit/Verify/VerifyCommand.cs | 46 +++++++++++-------- 4 files changed, 62 insertions(+), 25 deletions(-) diff --git a/src/Harp.Toolkit/Verify/ProtocolReference.cs b/src/Harp.Toolkit/Verify/ProtocolReference.cs index da09d9b..43af721 100644 --- a/src/Harp.Toolkit/Verify/ProtocolReference.cs +++ b/src/Harp.Toolkit/Verify/ProtocolReference.cs @@ -10,6 +10,11 @@ internal static class ProtocolReference /// public const string Commit = "11b584bdd2eb45a8b55ebc540512864ff0667326"; + /// + /// The major protocol version that specification text targets. + /// + public const int PrereleaseMajorVersion = 2; + /// /// Gets the abbreviated commit, for display where a link carries the full reference. /// diff --git a/src/Harp.Toolkit/Verify/ProtocolTarget.cs b/src/Harp.Toolkit/Verify/ProtocolTarget.cs index 621da97..58a3ed1 100644 --- a/src/Harp.Toolkit/Verify/ProtocolTarget.cs +++ b/src/Harp.Toolkit/Verify/ProtocolTarget.cs @@ -11,19 +11,17 @@ internal enum ProtocolScope internal readonly record struct ProtocolTarget(SemanticVersion? DeclaredVersion, bool PrereleaseRequested) { - const int PrereleaseMajorVersion = 2; - public ProtocolScope Scope { get { var major = DeclaredVersion.HasValue ? DeclaredVersion.GetValueOrDefault().Major : 0; - if (major > PrereleaseMajorVersion) + if (major > ProtocolReference.PrereleaseMajorVersion) return ProtocolScope.Unsupported; - return major == PrereleaseMajorVersion ? ProtocolScope.V2 : ProtocolScope.V1; + return major == ProtocolReference.PrereleaseMajorVersion ? ProtocolScope.V2 : ProtocolScope.V1; } } - public bool IncludePrerelease => Scope == ProtocolScope.V2 && PrereleaseRequested; + public bool IncludePrerelease => PrereleaseRequested; } diff --git a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs index 25ff5eb..b4e5e38 100644 --- a/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs +++ b/src/Harp.Toolkit/Verify/Suites/CoreRegisters/R_VERSION.cs @@ -38,6 +38,34 @@ public async Task IsNotWritable(VerifyConnection device) : "Version register should NOT be writable."); } + [HarpTest(Description = "Validates that Version register declares the protocol major version being checked.", Prerelease = true)] + public async Task AssertDeclaresCheckedMajorVersion(VerifyConnection device) + { + try + { + var reply = await device.CommandAsync(HarpCommand.ReadByte(Version.Address)); + var payload = reply.GetPayloadArray(); + if (payload.Length != Version.RegisterLength) + { + return new AssertionResult( + false, + $"Version returned {payload.Length} bytes, expected {Version.RegisterLength}."); + } + + var declared = Version.GetPayload(reply).ProtocolVersion; + return new AssertionResult( + declared.Major == ProtocolReference.PrereleaseMajorVersion, + x => x + ? $"Version declares protocol {declared}, matching the major version being checked." + : $"Version declares protocol {declared}, but these checks are against major " + + $"version {ProtocolReference.PrereleaseMajorVersion}."); + } + catch (Exception ex) + { + return new ErrorResult(ex); + } + } + [HarpTest(Description = "Reports the version information declared by the device.", Prerelease = true)] public async Task ReportVersionInformation(VerifyConnection device) { diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs index 11592ba..ab5bebf 100644 --- a/src/Harp.Toolkit/Verify/VerifyCommand.cs +++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs @@ -236,7 +236,7 @@ static string DescribeDeviceIdentity(DeviceIdentity identity, string portName) static string DescribeProtocolSelection(ProtocolTarget target) { - if (target.IncludePrerelease) + if (target.IncludePrerelease && target.Scope == ProtocolScope.V2) { return $"Checking against protocol version [bold]{GetDeclaredVersion(target)}[/], " + "which is not yet ratified."; @@ -249,39 +249,45 @@ static string DescribeProtocolSelection(ProtocolTarget target) static string GetCheckedVersion(ProtocolTarget target) { if (target.IncludePrerelease) - return $"{GetDeclaredVersion(target)}, which is not yet ratified"; + return $"v{ProtocolReference.PrereleaseMajorVersion}, which is not yet ratified"; - return target.Scope == ProtocolScope.V2 - ? "v1, since v2 is not yet ratified" - : "v1"; + return target.Scope == ProtocolScope.V1 + ? "v1" + : $"v1, since v{ProtocolReference.PrereleaseMajorVersion} is not yet ratified"; } static string GetProtocolNotice(ProtocolTarget target, int count) { + if (target.IncludePrerelease) + { + if (target.Scope == ProtocolScope.V2) + return $"Including {count} prerelease checks, which this device declares support for."; + + return $"Including {count} prerelease checks against protocol " + + $"v{ProtocolReference.PrereleaseMajorVersion}, which this device does not declare."; + } + if (target.Scope == ProtocolScope.Unsupported) { return $"This device declares protocol {GetDeclaredVersion(target)}, which this toolkit " + - "does not cover. The results below are against the v1 baseline only."; + $"does not cover, so only the v1 baseline applies. {GetRerunHint(count)}".TrimEnd(); } - if (target.Scope == ProtocolScope.V1) + if (!target.DeclaredVersion.HasValue) { - if (target.DeclaredVersion.HasValue) - return $"This device declares protocol {GetDeclaredVersion(target)}, so only the v1 baseline applies."; - - var recommendation = "Updating to a firmware that implements R_VERSION would let it be " + - "verified against the current protocol."; - return target.PrereleaseRequested - ? $"This device declares no protocol version, so --prerelease had no effect. {recommendation}" - : $"This device declares no protocol version. {recommendation}"; + return "This device declares no protocol version, so only the v1 baseline applies. " + + $"Implementing R_VERSION is the first step of a v{ProtocolReference.PrereleaseMajorVersion} " + + $"migration. {GetRerunHint(count)}".TrimEnd(); } - if (count == 0) - return string.Empty; + return GetRerunHint(count); + } - return target.PrereleaseRequested - ? $"Including {count} prerelease checks, which this device declares support for." - : $"{count} prerelease checks were not run. Rerun with --prerelease to include them."; + static string GetRerunHint(int count) + { + return count > 0 + ? $"{count} prerelease checks were not run. Rerun with --prerelease to include them." + : string.Empty; } static string GetResultMarkup(IResult result)