From e571ebf750610165f3f9866aec05744b0ac5faec Mon Sep 17 00:00:00 2001 From: james Date: Tue, 13 Oct 2026 20:37:37 +0100 Subject: [PATCH] Record repository, commit and rule-source provenance in results (#67) Results now say which repository, commit and branch were analysed, and which rule sources (and rules-cache commit) were used, so a report can be reproduced. JSON (validation-report.v1, additive optional fields, schemaVersion stays 1.0): repository, commit, branch, ruleSources[] (kind, location, commit). Omitted when unknown. SARIF: tool.driver.version is the CodeGuard version, run.versionControlProvenance carries repository/commit/branch when the repository URI is known, and rule sources are in the run property bag. Gathering lives in the CLI (ValidationProvenanceGatherer): CI variables for GitHub Actions, GitLab CI and Azure Pipelines win per field, and git fills the rest. Repository access and environment lookup are injected for tests. Remote URLs are normalised so credentials never reach a report. Reporters receive a ValidationProvenance and never shell out. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/IMPLEMENTATION_STATUS.md | 24 ++ docs/architecture/RESULT_CONTRACT.md | 30 ++- src/CodeGuard.Cli/Commands/ValidateCommand.cs | 23 +- .../Support/CliRepositoryContext.cs | 14 +- .../Support/GitRuleSourceSync.cs | 2 +- .../Support/RepositoryUrlNormalizer.cs | 56 +++++ .../Support/RulesSourceProvenance.cs | 8 + .../Support/ValidationProvenanceGatherer.cs | 102 +++++++++ .../Contracts/ValidationReportMapper.cs | 11 +- .../Contracts/ValidationReportV1.cs | 8 +- .../Json/JsonViolationReporter.cs | 8 +- .../Provenance/ValidationProvenance.cs | 21 ++ .../Sarif/SarifViolationReporter.cs | 54 +++++ .../Schemas/validation-report.v1.schema.json | 30 +++ .../RepositoryUrlNormalizerTests.cs | 34 +++ .../ValidationProvenanceGathererTests.cs | 211 ++++++++++++++++++ .../Reporting/SarifViolationReporterTests.cs | 65 ++++++ .../Reporting/ValidationReportSchemaTests.cs | 77 +++++++ 18 files changed, 760 insertions(+), 18 deletions(-) create mode 100644 src/CodeGuard.Cli/Support/RepositoryUrlNormalizer.cs create mode 100644 src/CodeGuard.Cli/Support/ValidationProvenanceGatherer.cs create mode 100644 src/CodeGuard.Reporting/Provenance/ValidationProvenance.cs create mode 100644 tests/CodeGuard.Cli.Tests/RepositoryUrlNormalizerTests.cs create mode 100644 tests/CodeGuard.Cli.Tests/ValidationProvenanceGathererTests.cs diff --git a/docs/architecture/IMPLEMENTATION_STATUS.md b/docs/architecture/IMPLEMENTATION_STATUS.md index 06f5e5d..95bccba 100644 --- a/docs/architecture/IMPLEMENTATION_STATUS.md +++ b/docs/architecture/IMPLEMENTATION_STATUS.md @@ -745,6 +745,30 @@ Decisions and gotchas: - `JsonSchema.Net` is referenced by `CodeGuard.Core.Tests` only, to validate output. Each schema is parsed once in a static `Lazy`, because `JsonSchema.FromText` throws on a duplicate `$id`. +### Post-v1 addition: repository, commit and rule-source provenance (#67) + +The JSON result (`validation-report.v1`) gained optional `repository`, `commit`, `branch` and +`ruleSources`. SARIF gained `tool.driver.version`, `run.versionControlProvenance` (when the repository +URI is known) and the rule sources in the run property bag `ruleSources`. Full field list: +`docs/architecture/RESULT_CONTRACT.md`. + +Decisions and gotchas: + +- Reporting never shells out. `CodeGuard.Cli.Support.ValidationProvenanceGatherer` gathers the + values and passes a `CodeGuard.Reporting.Provenance.ValidationProvenance` into the JSON and SARIF + reporter constructors. Repository access and environment lookup are injected (`IGitCommandRunner`, + an env lookup function) so precedence is unit-tested. +- Precedence is per field: a CI variable wins, and the repository fills in what CI does not provide. + GitHub uses `GITHUB_HEAD_REF` before `GITHUB_REF_NAME`, so a PR reports its source branch rather + than `N/merge`. `GITHUB_SHA` on a PR is the merge commit, not the PR head. +- Remotes are always normalised by `RepositoryUrlNormalizer`, including CI-provided URLs, because + Azure and GitLab can embed credentials. No raw remote reaches a report. +- SARIF `versionControlProvenance` requires `repositoryUri`, so a commit with no known remote is not + written there. The commit still appears in the JSON output. +- The process helper is `GitRuleSourceSync.RunCore`, now `internal`, so provenance and rule-cache + sync share one process runner. A missing git binary or a non-repository directory yields nulls and + never fails validation. + ## The 11 starter rules All under `rules/`, all illustrative (`Contoso.*` namespace, `illustrative: true`), matching the diff --git a/docs/architecture/RESULT_CONTRACT.md b/docs/architecture/RESULT_CONTRACT.md index ccb32bd..4a9c459 100644 --- a/docs/architecture/RESULT_CONTRACT.md +++ b/docs/architecture/RESULT_CONTRACT.md @@ -50,7 +50,13 @@ object has `additionalProperties: false`, so a consumer can rely on the set of k } ], "evaluationErrors": [], - "analysisWarnings": [] + "analysisWarnings": [], + "repository": "https://github.com/acme/app", + "commit": "9f1c2e4b7a0d3c5e6f8a1b2c3d4e5f6a7b8c9d0e", + "branch": "main", + "ruleSources": [ + { "kind": "git", "location": "https://github.com/acme/codeguard-rules", "commit": "8f31c7a0e1d2c3b4a5f6e7d8c9b0a1f2e3d4c5b6" } + ] } ``` @@ -121,6 +127,22 @@ The earlier output was the internal `ValidationResult` serialised directly. Cons ## Not yet in the contract -Planned by the roadmap but not part of v1: `rulesVersion`, `repository`, `commit`, `branch`, a -`timestamp` separate from `evaluatedAtUtc`, and per-run size metrics such as projects and types -analysed. These need the run context to be passed into the reporter. +Planned by the roadmap but not part of v1: `rulesVersion`, a `timestamp` separate from +`evaluatedAtUtc`, and per-run size metrics such as projects and types analysed. These need the run +context to be passed into the reporter. + +## Provenance (optional fields) + +These were added to v1 as optional fields, so `schemaVersion` stays `1.0`. They are omitted when +unknown, and omitted entirely when the caller supplies no provenance (for example a library caller). + +| Field | Type | Meaning | +|---|---|---| +| `repository` | string | Normalised remote URL of the analysed repository. Userinfo (`user:token@`), query and fragment are removed, and a trailing `.git` is dropped. scp-style remotes become `ssh://host/path`. | +| `commit` | string | Commit SHA of the analysed checkout. | +| `branch` | string | Branch of the analysed checkout. Omitted for a detached HEAD outside CI. | +| `ruleSources` | array | One entry per rules location the run used: `kind` (`git` or `directory`), `location` (git URL with credentials removed, or a directory path), and `commit` (HEAD of the rules cache, git only). | + +Values come from the CI environment when a known variable is set (GitHub Actions, GitLab CI, Azure +Pipelines), and otherwise from `git` in the repository root, field by field. Rule-source `commit` +is read from the rules cache clone, so it identifies the exact rules used. diff --git a/src/CodeGuard.Cli/Commands/ValidateCommand.cs b/src/CodeGuard.Cli/Commands/ValidateCommand.cs index 1699800..135187b 100644 --- a/src/CodeGuard.Cli/Commands/ValidateCommand.cs +++ b/src/CodeGuard.Cli/Commands/ValidateCommand.cs @@ -11,6 +11,7 @@ using CodeGuard.Reporting.Contracts; using CodeGuard.Reporting.Html; using CodeGuard.Reporting.Json; +using CodeGuard.Reporting.Provenance; using CodeGuard.Reporting.Sarif; using CodeGuard.RuleModel.Rules; using Microsoft.Extensions.Logging; @@ -208,8 +209,10 @@ public static Command Build() consoleOutputRedirected: Console.IsOutputRedirected, noColorEnvVar: Environment.GetEnvironmentVariable("NO_COLOR")); + var provenance = ValidationProvenanceGatherer.Default.Gather(context.RepoRoot, context.RuleSources); + var reporter = CreateReporter( - parseResult.GetValue(formatOption)!, useColor, totalStopwatch.Elapsed); + parseResult.GetValue(formatOption)!, useColor, totalStopwatch.Elapsed, provenance); logger.LogDebug("Writing {Format} report to {Destination}", parseResult.GetValue(formatOption), outputPath ?? "stdout"); @@ -264,14 +267,18 @@ private static ValidationResult ApplySeverityThreshold(ValidationResult result, } // durationMs in the JSON report is measured up to this point, i.e. just before the report is written. - private static IViolationReporter CreateReporter(string format, bool useColor, TimeSpan duration) => format switch + private static IViolationReporter CreateReporter( + string format, bool useColor, TimeSpan duration, ValidationProvenance provenance) { - "json" => new JsonViolationReporter( - ValidationReportMapper.ResolveToolVersion(typeof(ValidateCommand).Assembly), duration), - "sarif" => new SarifViolationReporter(), - "html" => new HtmlViolationReporter(), - _ => new ConsoleViolationReporter(useColor) - }; + var toolVersion = ValidationReportMapper.ResolveToolVersion(typeof(ValidateCommand).Assembly); + return format switch + { + "json" => new JsonViolationReporter(toolVersion, duration, provenance), + "sarif" => new SarifViolationReporter(provenance, toolVersion), + "html" => new HtmlViolationReporter(), + _ => new ConsoleViolationReporter(useColor) + }; + } private static Severity ParseSeverity(string value) => value switch { diff --git a/src/CodeGuard.Cli/Support/CliRepositoryContext.cs b/src/CodeGuard.Cli/Support/CliRepositoryContext.cs index 6951ccf..9eaea68 100644 --- a/src/CodeGuard.Cli/Support/CliRepositoryContext.cs +++ b/src/CodeGuard.Cli/Support/CliRepositoryContext.cs @@ -33,6 +33,9 @@ public sealed class CliRepositoryContext public GlobalSettings? GlobalSettings { get; init; } public ILoggerFactory? LoggerFactory { get; init; } + /// The rules locations the run is evaluated against, for result provenance. + public IReadOnlyList RuleSources { get; init; } = []; + /// /// Overrides where tier 4 looks for a prior `setup` run's settings.yml and git cache /// (default: the real OS app-data root, ). Exists @@ -61,12 +64,14 @@ public static CliRepositoryContext Resolve( ? RulesSourceProvenance.RepositoryConfig : RulesSourceProvenance.Default; + RuleSourceReference? explicitSource = null; if (rulesSource is not null) { logger.LogDebug("Using ad-hoc --rules-source override: {RulesSource} (branch {Branch})", rulesSource, branch ?? "(default)"); var overridePath = RuleSourceResolver.ResolveToLocalPath( rulesSource, branch, cacheRootOverride: globalSettingsRoot, logger: loggerFactory?.CreateLogger(typeof(RuleSourceResolver))); config = WithRules(config, [overridePath]); + explicitSource = new RuleSourceReference(RuleSourceResolver.DetectKind(rulesSource), rulesSource, overridePath); } var layout = new RepositoryDiscovery().Resolve(repoRoot, config); @@ -85,6 +90,7 @@ public static CliRepositoryContext Resolve( config = WithRules(config, [globalPath]); layout = new RepositoryDiscovery().Resolve(repoRoot, config); provenance = RulesSourceProvenance.GlobalSettings; + explicitSource = new RuleSourceReference(globalSettings.Kind, globalSettings.Location, globalPath); } } @@ -92,6 +98,11 @@ public static CliRepositoryContext Resolve( "Resolved rules source: provenance={Provenance}, repoRoot={RepoRoot}, rulesPaths={RulesPathCount}", provenance, repoRoot, layout.RulesPaths.Count); + // Repo-config and default rules are plain local directories; an explicit source is the one entry above. + IReadOnlyList ruleSources = explicitSource is not null + ? [explicitSource] + : layout.RulesPaths.Select(p => new RuleSourceReference(RuleSourceKind.Directory, p, p)).ToList(); + return new CliRepositoryContext { RepoRoot = repoRoot, @@ -99,7 +110,8 @@ public static CliRepositoryContext Resolve( RulesProvenance = provenance, ConfigFilePath = configFilePath, GlobalSettings = globalSettings, - LoggerFactory = loggerFactory + LoggerFactory = loggerFactory, + RuleSources = ruleSources }; } diff --git a/src/CodeGuard.Cli/Support/GitRuleSourceSync.cs b/src/CodeGuard.Cli/Support/GitRuleSourceSync.cs index 19b894c..df2e31d 100644 --- a/src/CodeGuard.Cli/Support/GitRuleSourceSync.cs +++ b/src/CodeGuard.Cli/Support/GitRuleSourceSync.cs @@ -107,7 +107,7 @@ private static void Run(IReadOnlyList args, string? workingDirectory, IL } } - private static (int ExitCode, string Stdout, string Stderr) RunCore(IReadOnlyList args, string? workingDirectory) + internal static (int ExitCode, string Stdout, string Stderr) RunCore(IReadOnlyList args, string? workingDirectory) { var startInfo = new ProcessStartInfo("git") { diff --git a/src/CodeGuard.Cli/Support/RepositoryUrlNormalizer.cs b/src/CodeGuard.Cli/Support/RepositoryUrlNormalizer.cs new file mode 100644 index 0000000..0eeaeb7 --- /dev/null +++ b/src/CodeGuard.Cli/Support/RepositoryUrlNormalizer.cs @@ -0,0 +1,56 @@ +using System.Text.RegularExpressions; + +namespace CodeGuard.Cli.Support; + +/// +/// Turns a git remote (URL or scp-style user@host:path) into a stable identity that is safe to +/// write into a report: userinfo (user:token@) is always removed, query/fragment dropped, a +/// trailing / and .git stripped, and scp-style remotes rewritten as ssh://host/path. +/// Credentials must never reach a report, so this runs on every remote before it is stored. +/// +public static partial class RepositoryUrlNormalizer +{ + // Authority ends at the last '@' before the first '/', so a password containing '@' is still removed. + [GeneratedRegex(@"^(?[a-zA-Z][a-zA-Z0-9+.\-]*://)[^/]*@")] + private static partial Regex UserInfo(); + + // Requires a host of 2+ characters so Windows drive letters (C:\repo) are not mistaken for hosts. + [GeneratedRegex(@"^(?:[^@/\s]+@)?(?[^:/\s]{2,}):(?.+)$")] + private static partial Regex ScpLike(); + + public static string? Normalize(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + { + return null; + } + + var value = raw.Trim(); + + if (value.Contains("://", StringComparison.Ordinal)) + { + value = UserInfo().Replace(value, "${scheme}"); + var cut = value.IndexOfAny(['?', '#']); + if (cut >= 0) + { + value = value[..cut]; + } + } + else + { + var match = ScpLike().Match(value); + if (match.Success) + { + value = $"ssh://{match.Groups["host"].Value}/{match.Groups["path"].Value.TrimStart('/')}"; + } + } + + value = value.TrimEnd('/'); + if (value.EndsWith(".git", StringComparison.OrdinalIgnoreCase)) + { + value = value[..^4].TrimEnd('/'); + } + + return value.Length == 0 ? null : value; + } +} diff --git a/src/CodeGuard.Cli/Support/RulesSourceProvenance.cs b/src/CodeGuard.Cli/Support/RulesSourceProvenance.cs index 6aa8ffc..695d4ac 100644 --- a/src/CodeGuard.Cli/Support/RulesSourceProvenance.cs +++ b/src/CodeGuard.Cli/Support/RulesSourceProvenance.cs @@ -1,5 +1,13 @@ +using CodeGuard.Configuration.GlobalConfig; + namespace CodeGuard.Cli.Support; +/// +/// One configured rules location as the CLI resolved it: the source as the user wrote it (a git URL +/// or directory), and the local directory the rules were actually loaded from (the git cache for git). +/// +public sealed record RuleSourceReference(RuleSourceKind Kind, string Location, string LocalPath); + /// Which tier of 's precedence chain produced the rules paths. public enum RulesSourceProvenance { diff --git a/src/CodeGuard.Cli/Support/ValidationProvenanceGatherer.cs b/src/CodeGuard.Cli/Support/ValidationProvenanceGatherer.cs new file mode 100644 index 0000000..15b955d --- /dev/null +++ b/src/CodeGuard.Cli/Support/ValidationProvenanceGatherer.cs @@ -0,0 +1,102 @@ +using CodeGuard.Configuration.GlobalConfig; +using CodeGuard.Reporting.Provenance; + +namespace CodeGuard.Cli.Support; + +/// Runs `git` for provenance. Returns null on any failure so provenance can never fail a validation. +public interface IGitCommandRunner +{ + /// Trimmed stdout, or null when git is missing, the directory is not a repository, or the command fails. + string? TryRun(string workingDirectory, params string[] args); +} + +/// Shells out to the system git via the same process helper uses. +public sealed class ProcessGitCommandRunner : IGitCommandRunner +{ + public string? TryRun(string workingDirectory, params string[] args) + { + try + { + var (exitCode, stdout, _) = GitRuleSourceSync.RunCore(args, workingDirectory); + return exitCode == 0 ? stdout.Trim() : null; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // Covers git not being installed (Win32Exception) and a missing working directory. + return null; + } + } +} + +/// +/// Builds the for a run. Each field comes from the CI environment +/// when a known CI variable is set, and otherwise from git in the repository root, field by field. +/// Environment and git access are injected so the precedence rules can be unit-tested. +/// +public sealed class ValidationProvenanceGatherer +{ + private readonly IGitCommandRunner _git; + private readonly Func _environment; + + public ValidationProvenanceGatherer(IGitCommandRunner git, Func environment) + { + _git = git; + _environment = environment; + } + + public static ValidationProvenanceGatherer Default { get; } = + new(new ProcessGitCommandRunner(), name => Environment.GetEnvironmentVariable(name)); + + public ValidationProvenance Gather(string repoRoot, IReadOnlyList ruleSources) + { + var repository = RepositoryUrlNormalizer.Normalize(FirstNonEmpty( + GitHubRepositoryUrl(), Env("CI_PROJECT_URL"), Env("BUILD_REPOSITORY_URI"))); + var commit = FirstNonEmpty(Env("GITHUB_SHA"), Env("CI_COMMIT_SHA"), Env("BUILD_SOURCEVERSION")); + // GITHUB_HEAD_REF is the source branch of a pull request; GITHUB_REF_NAME would be "N/merge" there. + var branch = FirstNonEmpty( + Env("GITHUB_HEAD_REF"), Env("GITHUB_REF_NAME"), Env("CI_COMMIT_REF_NAME"), Env("BUILD_SOURCEBRANCHNAME")); + + repository ??= RepositoryUrlNormalizer.Normalize(_git.TryRun(repoRoot, "config", "--get", "remote.origin.url")); + commit ??= Nullify(_git.TryRun(repoRoot, "rev-parse", "HEAD")); + if (branch is null) + { + var headName = Nullify(_git.TryRun(repoRoot, "rev-parse", "--abbrev-ref", "HEAD")); + // "HEAD" means detached; there is no branch to report. + branch = headName is "HEAD" ? null : headName; + } + + return new ValidationProvenance( + Repository: repository, + Commit: commit, + Branch: branch, + RuleSources: ruleSources.Select(ToRuleSetSource).ToList()); + } + + private RuleSetSource ToRuleSetSource(RuleSourceReference source) => source.Kind switch + { + // The cache is a clone, so its HEAD pins the exact rules used. The location is the remote URL, not the cache path. + RuleSourceKind.Git => new RuleSetSource( + "git", + RepositoryUrlNormalizer.Normalize(source.Location) ?? source.Location, + Nullify(_git.TryRun(source.LocalPath, "rev-parse", "HEAD"))), + _ => new RuleSetSource("directory", source.LocalPath, null) + }; + + private string? GitHubRepositoryUrl() + { + var repository = Env("GITHUB_REPOSITORY"); + if (repository is null) + { + return null; + } + + var server = (Env("GITHUB_SERVER_URL") ?? "https://github.com").TrimEnd('/'); + return $"{server}/{repository}"; + } + + private string? Env(string name) => Nullify(_environment(name)); + + private static string? FirstNonEmpty(params string?[] values) => values.FirstOrDefault(v => v is not null); + + private static string? Nullify(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); +} diff --git a/src/CodeGuard.Reporting/Contracts/ValidationReportMapper.cs b/src/CodeGuard.Reporting/Contracts/ValidationReportMapper.cs index 06b3ec6..13684be 100644 --- a/src/CodeGuard.Reporting/Contracts/ValidationReportMapper.cs +++ b/src/CodeGuard.Reporting/Contracts/ValidationReportMapper.cs @@ -1,5 +1,6 @@ using System.Reflection; using CodeGuard.Core.Results; +using CodeGuard.Reporting.Provenance; using CodeGuard.RuleModel.Rules; namespace CodeGuard.Reporting.Contracts; @@ -9,7 +10,9 @@ public static class ValidationReportMapper { public const string SchemaVersion = "1.0"; - public static ValidationReportV1 ToV1(ValidationResult result, TimeSpan duration, string toolVersion) + /// Repository and rule-source provenance, when the caller gathered it. Null leaves those fields out. + public static ValidationReportV1 ToV1( + ValidationResult result, TimeSpan duration, string toolVersion, ValidationProvenance? provenance = null) { var violations = result.Violations.Select(MapViolation).ToList(); @@ -34,6 +37,12 @@ public static ValidationReportV1 ToV1(ValidationResult result, TimeSpan duration .ToList(), AnalysisWarnings: result.AnalysisWarnings .Select(w => new AnalysisWarningV1(w.Code, w.Message, w.Project, w.FilePath)) + .ToList(), + Repository: provenance?.Repository, + Commit: provenance?.Commit, + Branch: provenance?.Branch, + RuleSources: provenance?.RuleSources + .Select(r => new RuleSetSourceV1(r.Kind, r.Location, r.Commit)) .ToList()); } diff --git a/src/CodeGuard.Reporting/Contracts/ValidationReportV1.cs b/src/CodeGuard.Reporting/Contracts/ValidationReportV1.cs index 8b3fd17..e07542d 100644 --- a/src/CodeGuard.Reporting/Contracts/ValidationReportV1.cs +++ b/src/CodeGuard.Reporting/Contracts/ValidationReportV1.cs @@ -14,7 +14,11 @@ public sealed record ValidationReportV1( ValidationReportMetricsV1 Metrics, IReadOnlyList Violations, IReadOnlyList EvaluationErrors, - IReadOnlyList AnalysisWarnings); + IReadOnlyList AnalysisWarnings, + string? Repository = null, + string? Commit = null, + string? Branch = null, + IReadOnlyList? RuleSources = null); public sealed record ValidationReportMetricsV1( int RulesEvaluated, @@ -52,3 +56,5 @@ public sealed record AnalysisWarningV1( string Message, string? Project, string? FilePath); + +public sealed record RuleSetSourceV1(string Kind, string Location, string? Commit); diff --git a/src/CodeGuard.Reporting/Json/JsonViolationReporter.cs b/src/CodeGuard.Reporting/Json/JsonViolationReporter.cs index 8443d65..7b797cb 100644 --- a/src/CodeGuard.Reporting/Json/JsonViolationReporter.cs +++ b/src/CodeGuard.Reporting/Json/JsonViolationReporter.cs @@ -2,6 +2,7 @@ using System.Text.Json.Serialization; using CodeGuard.Core.Results; using CodeGuard.Reporting.Contracts; +using CodeGuard.Reporting.Provenance; namespace CodeGuard.Reporting.Json; @@ -20,20 +21,23 @@ public sealed class JsonViolationReporter : IViolationReporter private readonly string _toolVersion; private readonly TimeSpan _duration; + private readonly ValidationProvenance? _provenance; /// Reported as toolVersion. Defaults to this assembly's version. /// Reported as durationMs. Defaults to zero, i.e. not measured. - public JsonViolationReporter(string? toolVersion = null, TimeSpan duration = default) + /// Reported as repository/commit/branch/ruleSources. Omitted when null. + public JsonViolationReporter(string? toolVersion = null, TimeSpan duration = default, ValidationProvenance? provenance = null) { _toolVersion = toolVersion ?? ValidationReportMapper.ResolveToolVersion(typeof(JsonViolationReporter).Assembly); _duration = duration; + _provenance = provenance; } public string Format => "json"; public async Task WriteAsync(ValidationResult result, TextWriter writer, CancellationToken ct = default) { - var report = ValidationReportMapper.ToV1(result, _duration, _toolVersion); + var report = ValidationReportMapper.ToV1(result, _duration, _toolVersion, _provenance); var json = JsonSerializer.Serialize(report, Options); await writer.WriteLineAsync(json.AsMemory(), ct); } diff --git a/src/CodeGuard.Reporting/Provenance/ValidationProvenance.cs b/src/CodeGuard.Reporting/Provenance/ValidationProvenance.cs new file mode 100644 index 0000000..79551f3 --- /dev/null +++ b/src/CodeGuard.Reporting/Provenance/ValidationProvenance.cs @@ -0,0 +1,21 @@ +namespace CodeGuard.Reporting.Provenance; + +/// +/// Where a validation run came from: the analysed repository and the rule sets it was evaluated +/// against. Gathered by the CLI (git and CI environment) and handed to the reporters, which never +/// shell out themselves. Every field is optional; unknown values are null and omitted from output. +/// +/// Normalised remote URL, with any credentials or userinfo removed. +/// Commit SHA of the analysed checkout. +/// Branch name of the analysed checkout (null when detached and not known from CI). +/// Rule sets used, one per configured rules location. +public sealed record ValidationProvenance( + string? Repository, + string? Commit, + string? Branch, + IReadOnlyList RuleSources); + +/// "git" or "directory". +/// Normalised git URL (credentials removed) or local directory path. +/// HEAD of the rules cache for git sources; null for directories. +public sealed record RuleSetSource(string Kind, string Location, string? Commit); diff --git a/src/CodeGuard.Reporting/Sarif/SarifViolationReporter.cs b/src/CodeGuard.Reporting/Sarif/SarifViolationReporter.cs index e27b821..0726570 100644 --- a/src/CodeGuard.Reporting/Sarif/SarifViolationReporter.cs +++ b/src/CodeGuard.Reporting/Sarif/SarifViolationReporter.cs @@ -1,11 +1,27 @@ using Microsoft.CodeAnalysis.Sarif; using CodeGuard.Core.Results; +using CodeGuard.Reporting.Contracts; +using CodeGuard.Reporting.Provenance; using CodeGuard.RuleModel.Rules; namespace CodeGuard.Reporting.Sarif; public sealed class SarifViolationReporter : IViolationReporter { + private readonly ValidationProvenance? _provenance; + private readonly string _toolVersion; + + /// + /// Populates run.versionControlProvenance (when a repository URI is known) and the run property + /// bag's ruleSources. Null leaves both out. + /// + /// Reported as tool.driver.version. Defaults to this assembly's version. + public SarifViolationReporter(ValidationProvenance? provenance = null, string? toolVersion = null) + { + _provenance = provenance; + _toolVersion = toolVersion ?? ValidationReportMapper.ResolveToolVersion(typeof(SarifViolationReporter).Assembly); + } + public string Format => "sarif"; public async Task WriteAsync(ValidationResult result, TextWriter writer, CancellationToken ct = default) @@ -35,6 +51,7 @@ public async Task WriteAsync(ValidationResult result, TextWriter writer, Cancell Driver = new ToolComponent { Name = "codeguard", + Version = _toolVersion, Rules = rules } }, @@ -51,6 +68,8 @@ public async Task WriteAsync(ValidationResult result, TextWriter writer, Cancell ] }; + AddProvenance(run, _provenance); + var log = new SarifLog { Version = SarifVersion.Current, @@ -67,6 +86,41 @@ public async Task WriteAsync(ValidationResult result, TextWriter writer, Cancell await writer.WriteLineAsync(json.AsMemory(), ct); } + /// + /// Repository identity goes in versionControlProvenance, which SARIF requires to carry a + /// repositoryUri, so it is only written when the repository is known and parses as a URI. + /// Rule-set provenance has no SARIF home, so it goes in the run property bag. + /// + private static void AddProvenance(Run run, ValidationProvenance? provenance) + { + if (provenance is null) + { + return; + } + + if (provenance.Repository is not null && Uri.TryCreate(provenance.Repository, UriKind.Absolute, out var repositoryUri)) + { + run.VersionControlProvenance = + [ + new VersionControlDetails + { + RepositoryUri = repositoryUri, + RevisionId = provenance.Commit, + Branch = provenance.Branch + } + ]; + } + + run.SetProperty("ruleSources", provenance.RuleSources + .Select(r => new Dictionary + { + ["kind"] = r.Kind, + ["location"] = r.Location, + ["commit"] = r.Commit + }) + .ToList()); + } + private static Result ToSarifResult(Violation violation) { var result = new Result diff --git a/src/CodeGuard.Reporting/Schemas/validation-report.v1.schema.json b/src/CodeGuard.Reporting/Schemas/validation-report.v1.schema.json index f4441a9..1801428 100644 --- a/src/CodeGuard.Reporting/Schemas/validation-report.v1.schema.json +++ b/src/CodeGuard.Reporting/Schemas/validation-report.v1.schema.json @@ -53,6 +53,26 @@ "analysisWarnings": { "type": "array", "items": { "$ref": "#/$defs/analysisWarning" } + }, + "repository": { + "type": "string", + "minLength": 1, + "description": "Normalised remote URL of the analysed repository, with any credentials or userinfo removed. Omitted when unknown." + }, + "commit": { + "type": "string", + "minLength": 1, + "description": "Commit SHA of the analysed checkout. Omitted when unknown." + }, + "branch": { + "type": "string", + "minLength": 1, + "description": "Branch of the analysed checkout. Omitted when unknown (for example a detached HEAD outside CI)." + }, + "ruleSources": { + "type": "array", + "items": { "$ref": "#/$defs/ruleSetSource" }, + "description": "Rule sets the run was evaluated against. Omitted when the producer did not record rule-source provenance." } }, "$defs": { @@ -136,6 +156,16 @@ "project": { "type": "string" }, "filePath": { "type": "string" } } + }, + "ruleSetSource": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "location"], + "properties": { + "kind": { "type": "string", "enum": ["git", "directory"] }, + "location": { "type": "string", "minLength": 1, "description": "Normalised git URL (credentials removed) or local directory path." }, + "commit": { "type": "string", "minLength": 1, "description": "HEAD of the rules cache for git sources. Omitted for directories and when unknown." } + } } } } diff --git a/tests/CodeGuard.Cli.Tests/RepositoryUrlNormalizerTests.cs b/tests/CodeGuard.Cli.Tests/RepositoryUrlNormalizerTests.cs new file mode 100644 index 0000000..ce78dfa --- /dev/null +++ b/tests/CodeGuard.Cli.Tests/RepositoryUrlNormalizerTests.cs @@ -0,0 +1,34 @@ +using CodeGuard.Cli.Support; + +namespace CodeGuard.Cli.Tests; + +public class RepositoryUrlNormalizerTests +{ + [Theory] + [InlineData("https://github.com/acme/app.git", "https://github.com/acme/app")] + [InlineData("https://github.com/acme/app/", "https://github.com/acme/app")] + [InlineData("https://github.com/acme/app", "https://github.com/acme/app")] + [InlineData("https://user:ghp_secret@github.com/acme/app.git", "https://github.com/acme/app")] + [InlineData("https://x-access-token@github.com/acme/app", "https://github.com/acme/app")] + [InlineData("https://user:p@ss@host.example.com/org/repo.git", "https://host.example.com/org/repo")] + [InlineData("https://host.example.com/org/repo.git?token=abc#frag", "https://host.example.com/org/repo")] + [InlineData("ssh://git@host.example.com:2222/org/repo.git", "ssh://host.example.com:2222/org/repo")] + [InlineData("git@github.com:acme/app.git", "ssh://github.com/acme/app")] + [InlineData("github.com:acme/app", "ssh://github.com/acme/app")] + [InlineData(" https://github.com/acme/app.git ", "https://github.com/acme/app")] + [InlineData("/srv/git/repo.git", "/srv/git/repo")] + [InlineData("C:\\repos\\app", "C:\\repos\\app")] + public void Normalize_RemovesCredentialsAndDecoration(string raw, string expected) + { + Assert.Equal(expected, RepositoryUrlNormalizer.Normalize(raw)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Normalize_EmptyInput_ReturnsNull(string? raw) + { + Assert.Null(RepositoryUrlNormalizer.Normalize(raw)); + } +} diff --git a/tests/CodeGuard.Cli.Tests/ValidationProvenanceGathererTests.cs b/tests/CodeGuard.Cli.Tests/ValidationProvenanceGathererTests.cs new file mode 100644 index 0000000..ae15caf --- /dev/null +++ b/tests/CodeGuard.Cli.Tests/ValidationProvenanceGathererTests.cs @@ -0,0 +1,211 @@ +using CodeGuard.Cli.Support; +using CodeGuard.Configuration.GlobalConfig; + +namespace CodeGuard.Cli.Tests; + +public class ValidationProvenanceGathererTests +{ + private const string RepoRoot = "/work/repo"; + private const string RulesCache = "/cache/rules"; + + private static readonly string[] OriginUrl = ["config", "--get", "remote.origin.url"]; + private static readonly string[] HeadSha = ["rev-parse", "HEAD"]; + private static readonly string[] HeadBranch = ["rev-parse", "--abbrev-ref", "HEAD"]; + + [Fact] + public void Gather_OutsideGit_ReturnsNullsAndDirectorySource() + { + var gatherer = new ValidationProvenanceGatherer(new FakeGit(), _ => null); + + var result = gatherer.Gather(RepoRoot, [new RuleSourceReference(RuleSourceKind.Directory, "/r/rules", "/r/rules")]); + + Assert.Null(result.Repository); + Assert.Null(result.Commit); + Assert.Null(result.Branch); + var source = Assert.Single(result.RuleSources); + Assert.Equal("directory", source.Kind); + Assert.Equal("/r/rules", source.Location); + Assert.Null(source.Commit); + } + + [Fact] + public void Gather_FromGit_ReadsRemoteCommitAndBranch() + { + var git = new FakeGit + { + [(RepoRoot, OriginUrl)] = "git@github.com:acme/app.git\n", + [(RepoRoot, HeadSha)] = "abc123\n", + [(RepoRoot, HeadBranch)] = "feature/x\n" + }; + + var result = new ValidationProvenanceGatherer(git, _ => null).Gather(RepoRoot, []); + + Assert.Equal("ssh://github.com/acme/app", result.Repository); + Assert.Equal("abc123", result.Commit); + Assert.Equal("feature/x", result.Branch); + } + + [Fact] + public void Gather_DetachedHeadWithoutCi_ReportsNoBranch() + { + var git = new FakeGit { [(RepoRoot, HeadBranch)] = "HEAD\n" }; + + var result = new ValidationProvenanceGatherer(git, _ => null).Gather(RepoRoot, []); + + Assert.Null(result.Branch); + } + + [Fact] + public void Gather_GitHubActions_EnvironmentOverridesGit() + { + var git = new FakeGit + { + [(RepoRoot, OriginUrl)] = "https://git.local/old.git", + [(RepoRoot, HeadSha)] = "localsha", + [(RepoRoot, HeadBranch)] = "HEAD" + }; + var env = Env( + ("GITHUB_REPOSITORY", "acme/app"), + ("GITHUB_SERVER_URL", "https://github.com/"), + ("GITHUB_SHA", "ci-sha"), + ("GITHUB_REF_NAME", "42/merge"), + ("GITHUB_HEAD_REF", "fix/thing")); + + var result = new ValidationProvenanceGatherer(git, env).Gather(RepoRoot, []); + + Assert.Equal("https://github.com/acme/app", result.Repository); + Assert.Equal("ci-sha", result.Commit); + Assert.Equal("fix/thing", result.Branch); + } + + [Fact] + public void Gather_GitHubActions_FallsBackToRefNameWithoutHeadRef() + { + var env = Env(("GITHUB_REPOSITORY", "acme/app"), ("GITHUB_REF_NAME", "main")); + + var result = new ValidationProvenanceGatherer(new FakeGit(), env).Gather(RepoRoot, []); + + Assert.Equal("https://github.com/acme/app", result.Repository); + Assert.Equal("main", result.Branch); + } + + [Fact] + public void Gather_GitLabCi_UsesProjectUrlCommitAndRefName() + { + var env = Env( + ("CI_PROJECT_URL", "https://gitlab.example.com/group/project"), + ("CI_COMMIT_SHA", "gl-sha"), + ("CI_COMMIT_REF_NAME", "release")); + + var result = new ValidationProvenanceGatherer(new FakeGit(), env).Gather(RepoRoot, []); + + Assert.Equal("https://gitlab.example.com/group/project", result.Repository); + Assert.Equal("gl-sha", result.Commit); + Assert.Equal("release", result.Branch); + } + + [Fact] + public void Gather_AzurePipelines_UsesBuildVariables() + { + var env = Env( + ("BUILD_REPOSITORY_URI", "https://dev.azure.com/org/project/_git/repo"), + ("BUILD_SOURCEVERSION", "az-sha"), + ("BUILD_SOURCEBRANCHNAME", "develop")); + + var result = new ValidationProvenanceGatherer(new FakeGit(), env).Gather(RepoRoot, []); + + Assert.Equal("https://dev.azure.com/org/project/_git/repo", result.Repository); + Assert.Equal("az-sha", result.Commit); + Assert.Equal("develop", result.Branch); + } + + [Fact] + public void Gather_CiUrlWithCredentials_IsStripped() + { + var env = Env(("CI_PROJECT_URL", "https://oauth2:glpat-secret@gitlab.example.com/group/project.git")); + + var result = new ValidationProvenanceGatherer(new FakeGit(), env).Gather(RepoRoot, []); + + Assert.Equal("https://gitlab.example.com/group/project", result.Repository); + Assert.DoesNotContain("secret", result.Repository, StringComparison.Ordinal); + } + + [Fact] + public void Gather_GitRemoteWithCredentials_IsStripped() + { + var git = new FakeGit { [(RepoRoot, OriginUrl)] = "https://user:tok@dev.example.com/org/repo.git" }; + + var result = new ValidationProvenanceGatherer(git, _ => null).Gather(RepoRoot, []); + + Assert.Equal("https://dev.example.com/org/repo", result.Repository); + } + + [Fact] + public void Gather_GitRuleSource_ReportsRemoteAndCacheCommit() + { + var git = new FakeGit { [(RulesCache, HeadSha)] = "rules-sha" }; + var source = new RuleSourceReference(RuleSourceKind.Git, "https://u:p@rules.example.com/acme/rules.git", RulesCache); + + var result = new ValidationProvenanceGatherer(git, _ => null).Gather(RepoRoot, [source]); + + var rule = Assert.Single(result.RuleSources); + Assert.Equal("git", rule.Kind); + Assert.Equal("https://rules.example.com/acme/rules", rule.Location); + Assert.Equal("rules-sha", rule.Commit); + } + + [Fact] + public void Gather_GitRuleSourceWithoutGit_KeepsLocationWithNullCommit() + { + var source = new RuleSourceReference(RuleSourceKind.Git, "https://rules.example.com/acme/rules.git", RulesCache); + + var result = new ValidationProvenanceGatherer(new FakeGit(), _ => null).Gather(RepoRoot, [source]); + + var rule = Assert.Single(result.RuleSources); + Assert.Equal("https://rules.example.com/acme/rules", rule.Location); + Assert.Null(rule.Commit); + } + + [Fact] + public void ProcessGitCommandRunner_NonRepositoryDirectory_ReturnsNull() + { + var notARepo = Path.Combine(Path.GetTempPath(), "codeguard-provenance-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(notARepo); + try + { + Assert.Null(new ProcessGitCommandRunner().TryRun(notARepo, "rev-parse", "HEAD")); + } + finally + { + Directory.Delete(notARepo, recursive: true); + } + } + + [Fact] + public void ProcessGitCommandRunner_MissingDirectory_ReturnsNull() + { + var missing = Path.Combine(Path.GetTempPath(), "codeguard-missing-" + Guid.NewGuid().ToString("N")); + + Assert.Null(new ProcessGitCommandRunner().TryRun(missing, "rev-parse", "HEAD")); + } + + private static Func Env(params (string Name, string Value)[] values) + { + var map = values.ToDictionary(v => v.Name, v => v.Value, StringComparer.Ordinal); + return name => map.GetValueOrDefault(name); + } + + /// Keyed by (working directory, git arguments). Unmapped calls behave as "git fails". + private sealed class FakeGit : IGitCommandRunner + { + private readonly Dictionary<(string Dir, string Args), string> _responses = new(); + + public string? this[(string Dir, string[] Args) key] + { + set => _responses[(key.Dir, string.Join(' ', key.Args))] = value!; + } + + public string? TryRun(string workingDirectory, params string[] args) => + _responses.TryGetValue((workingDirectory, string.Join(' ', args)), out var output) ? output : null; + } +} diff --git a/tests/CodeGuard.Core.Tests/Reporting/SarifViolationReporterTests.cs b/tests/CodeGuard.Core.Tests/Reporting/SarifViolationReporterTests.cs index 67d6df9..2f55fbd 100644 --- a/tests/CodeGuard.Core.Tests/Reporting/SarifViolationReporterTests.cs +++ b/tests/CodeGuard.Core.Tests/Reporting/SarifViolationReporterTests.cs @@ -1,5 +1,6 @@ using System.Text.Json; using CodeGuard.Core.Results; +using CodeGuard.Reporting.Provenance; using CodeGuard.Reporting.Sarif; using CodeGuard.RuleModel.Rules; @@ -7,6 +8,70 @@ namespace CodeGuard.Core.Tests.Reporting; public class SarifViolationReporterTests { + private static ValidationResult EmptyPassingResult() => new( + ValidationStatus.Passed, RulesEvaluated: 0, RulesPassed: 0, RulesFailed: 0, RulesErrored: 0, + Violations: [], EvaluationErrors: [], EvaluatedAtUtc: DateTimeOffset.UtcNow); + + [Fact] + public async Task WriteAsync_WithProvenance_WritesVersionControlDetailsAndRuleSources() + { + var provenance = new ValidationProvenance( + Repository: "https://github.com/acme/app", + Commit: "abc123", + Branch: "main", + RuleSources: [new RuleSetSource("git", "https://rules.example.com/acme/rules", "rules-sha")]); + + var writer = new StringWriter(); + await new SarifViolationReporter(provenance, "1.2.3").WriteAsync(EmptyPassingResult(), writer); + + using var document = JsonDocument.Parse(writer.ToString()); + var run = document.RootElement.GetProperty("runs")[0]; + + var driver = run.GetProperty("tool").GetProperty("driver"); + Assert.Equal("1.2.3", driver.GetProperty("version").GetString()); + + var vcs = run.GetProperty("versionControlProvenance")[0]; + Assert.Equal("https://github.com/acme/app", vcs.GetProperty("repositoryUri").GetString()); + Assert.Equal("abc123", vcs.GetProperty("revisionId").GetString()); + Assert.Equal("main", vcs.GetProperty("branch").GetString()); + + var ruleSource = run.GetProperty("properties").GetProperty("ruleSources")[0]; + Assert.Equal("git", ruleSource.GetProperty("kind").GetString()); + Assert.Equal("https://rules.example.com/acme/rules", ruleSource.GetProperty("location").GetString()); + Assert.Equal("rules-sha", ruleSource.GetProperty("commit").GetString()); + } + + [Fact] + public async Task WriteAsync_WithoutProvenance_OmitsVersionControlDetails() + { + var writer = new StringWriter(); + await new SarifViolationReporter().WriteAsync(EmptyPassingResult(), writer); + + using var document = JsonDocument.Parse(writer.ToString()); + var run = document.RootElement.GetProperty("runs")[0]; + + Assert.False(run.TryGetProperty("versionControlProvenance", out _)); + Assert.False(run.TryGetProperty("properties", out _)); + Assert.False(string.IsNullOrEmpty(run.GetProperty("tool").GetProperty("driver").GetProperty("version").GetString())); + } + + [Fact] + public async Task WriteAsync_WithoutRepositoryUri_OmitsVersionControlDetailsButKeepsRuleSources() + { + var provenance = new ValidationProvenance( + Repository: null, Commit: "abc123", Branch: null, + RuleSources: [new RuleSetSource("directory", "/repo/rules", null)]); + + var writer = new StringWriter(); + await new SarifViolationReporter(provenance, "1.2.3").WriteAsync(EmptyPassingResult(), writer); + + using var document = JsonDocument.Parse(writer.ToString()); + var run = document.RootElement.GetProperty("runs")[0]; + + Assert.False(run.TryGetProperty("versionControlProvenance", out _)); + Assert.Equal("/repo/rules", run.GetProperty("properties").GetProperty("ruleSources")[0].GetProperty("location").GetString()); + } + [Fact] public async Task WriteAsync_ProducesValidSarifLog() { diff --git a/tests/CodeGuard.Core.Tests/Reporting/ValidationReportSchemaTests.cs b/tests/CodeGuard.Core.Tests/Reporting/ValidationReportSchemaTests.cs index 72da757..dd78130 100644 --- a/tests/CodeGuard.Core.Tests/Reporting/ValidationReportSchemaTests.cs +++ b/tests/CodeGuard.Core.Tests/Reporting/ValidationReportSchemaTests.cs @@ -1,6 +1,7 @@ using System.Text.Json; using CodeGuard.Core.Results; using CodeGuard.Reporting.Json; +using CodeGuard.Reporting.Provenance; using CodeGuard.RuleModel.Rules; using Json.Schema; @@ -103,6 +104,82 @@ public void Schema_RejectsUnknownTopLevelProperty() Assert.False(results.IsValid); } + [Fact] + public async Task Output_WithProvenance_ConformsToSchemaAndCarriesFields() + { + var result = new ValidationResult( + ValidationStatus.Passed, RulesEvaluated: 1, RulesPassed: 1, RulesFailed: 0, RulesErrored: 0, + Violations: [], EvaluationErrors: [], EvaluatedAtUtc: DateTimeOffset.UtcNow); + var provenance = new ValidationProvenance( + Repository: "https://github.com/acme/app", + Commit: "abc123", + Branch: "main", + RuleSources: + [ + new RuleSetSource("git", "https://rules.example.com/acme/rules", "rules-sha"), + new RuleSetSource("directory", "/repo/rules", null) + ]); + + var writer = new StringWriter(); + await new JsonViolationReporter("1.2.3", TimeSpan.Zero, provenance).WriteAsync(result, writer); + var output = writer.ToString(); + + AssertConforms(output); + using var document = JsonDocument.Parse(output); + var root = document.RootElement; + Assert.Equal("https://github.com/acme/app", root.GetProperty("repository").GetString()); + Assert.Equal("abc123", root.GetProperty("commit").GetString()); + Assert.Equal("main", root.GetProperty("branch").GetString()); + var sources = root.GetProperty("ruleSources"); + Assert.Equal("git", sources[0].GetProperty("kind").GetString()); + Assert.Equal("rules-sha", sources[0].GetProperty("commit").GetString()); + Assert.False(sources[1].TryGetProperty("commit", out _)); + } + + [Fact] + public async Task Output_WithUnknownProvenance_OmitsFields() + { + var result = new ValidationResult( + ValidationStatus.Passed, RulesEvaluated: 0, RulesPassed: 0, RulesFailed: 0, RulesErrored: 0, + Violations: [], EvaluationErrors: [], EvaluatedAtUtc: DateTimeOffset.UtcNow); + var provenance = new ValidationProvenance(null, null, null, []); + + var writer = new StringWriter(); + await new JsonViolationReporter("1.2.3", TimeSpan.Zero, provenance).WriteAsync(result, writer); + var output = writer.ToString(); + + AssertConforms(output); + using var document = JsonDocument.Parse(output); + Assert.False(document.RootElement.TryGetProperty("repository", out _)); + Assert.False(document.RootElement.TryGetProperty("commit", out _)); + Assert.False(document.RootElement.TryGetProperty("branch", out _)); + Assert.Empty(document.RootElement.GetProperty("ruleSources").EnumerateArray()); + } + + [Fact] + public void Schema_RejectsCredentialFieldsInRuleSource() + { + const string document = """ + { + "schemaVersion": "1.0", "toolVersion": "1.0.0", "status": "passed", + "evaluatedAtUtc": "2026-01-01T00:00:00+00:00", "durationMs": 0, + "metrics": { + "rulesEvaluated": 0, "rulesPassed": 0, "rulesFailed": 0, "rulesErrored": 0, + "violationCount": 0, + "violationsBySeverity": { "info": 0, "warning": 0, "error": 0, "critical": 0 }, + "evaluationErrorCount": 0, "analysisWarningCount": 0 + }, + "violations": [], "evaluationErrors": [], "analysisWarnings": [], + "ruleSources": [ { "kind": "git", "location": "https://u:p@host/x", "token": "p" } ] + } + """; + + using var parsed = JsonDocument.Parse(document); + var results = ReportSchema.Value.Evaluate(parsed.RootElement, new EvaluationOptions { OutputFormat = OutputFormat.List }); + + Assert.False(results.IsValid); + } + private static async Task WriteAsync(ValidationResult result, TimeSpan duration) { var writer = new StringWriter();