diff --git a/docs/architecture/IMPLEMENTATION_STATUS.md b/docs/architecture/IMPLEMENTATION_STATUS.md index 3ef1c14..7f52bd3 100644 --- a/docs/architecture/IMPLEMENTATION_STATUS.md +++ b/docs/architecture/IMPLEMENTATION_STATUS.md @@ -476,11 +476,13 @@ without needing Phase 2's rule `metadata` first. containing them (like this repo's own `examples/rules/`, all illustrative) isn't itself a problem. Missing provenance is correctly not implemented - still blocked on §6/§19's unimplemented `metadata` field. - - Tier 2 "unreachable rules": a top-level assertion whose `CapabilityDescriptor.AppliesTo` doesn't - include the target selector's `Produces` kind. Doesn't recurse into `must_all_match`/ - `must_any_match`/`must_none_match`'s nested `assertions:` - those objects aren't recoverable from - the parsed model (only `Kind` survives parsing), only from the raw document, and this check - works from the parsed model since `Kind` is all it needs for the top level. + - Tier 2 "unreachable rules": an assertion whose `CapabilityDescriptor.AppliesTo` doesn't include + the candidate kind it is checked against - the target selector's `Produces` for top-level + assertions and `when:` leaves, and the nested `selector:`'s `Produces` for assertions inside + `must_all_match`/`must_any_match`/`must_none_match`'s `assertions:` (recursively). Top-level + uses the parsed model; nested and `when:` use the raw document (`RuleSetAnalyzer`'s + `NestedAssertionScanner`), since only `Kind` survives parsing. Findings carry a `Location` path + (e.g. `assertions[0].assertions[1]`, `when.and[1]`). - Tier 2 "exact-duplicate rules": needs the raw source document instead (`RuleFileLoader.ReadDocument`), for the reason above once parameter values matter - a `Canonicalize` helper recursively key-sorts each rule's `target`+`assertions` (or `analyzer`) before grouping, so two rules diff --git a/docs/roadmap/HIGH_LEVEL_AI_ASSISTING.md b/docs/roadmap/HIGH_LEVEL_AI_ASSISTING.md index cae3e97..cdfa944 100644 --- a/docs/roadmap/HIGH_LEVEL_AI_ASSISTING.md +++ b/docs/roadmap/HIGH_LEVEL_AI_ASSISTING.md @@ -638,10 +638,10 @@ them being treated as one piece of work. They split into three tiers: the target selector's `Produces` kind. Assertions with an empty `AppliesTo` (the quantifier/ existence kinds — `must_all_match`/`must_any_match`/`must_none_match`/`must_have_count`/ `must_exist`/`must_not_exist`, which evaluate a nested selector rather than the outer candidate) - are never flagged, by design. **Known limitation, not fixed**: this only checks the rule's - top-level assertions — it doesn't recurse into those quantifier assertions' own nested - `assertions:`, because the nested selector/assertion objects aren't recoverable from the parsed - model (only `Kind` survives parsing, the same limitation §13 works around for parameter values). + are never flagged, by design. Assertions nested in a quantifier's `assertions:` are checked + against that quantifier's `selector:`, and `when:` leaves against the target, via the raw source + document (the parsed model keeps only `Kind`, the same limitation §13 works around for parameter + values). Resolved — previously a known limitation. * Exact-duplicate rules (identical `target`+`assertions`, or identical `analyzer`) — reads the raw source document via `RuleFileLoader.ReadDocument` rather than the parsed model, for the same reason, and canonicalizes (recursively key-sorts) it first so two rules that differ only in diff --git a/src/CodeGuard.Cli/Support/RuleAnalysisReportWriter.cs b/src/CodeGuard.Cli/Support/RuleAnalysisReportWriter.cs index 46a77df..ad8ae15 100644 --- a/src/CodeGuard.Cli/Support/RuleAnalysisReportWriter.cs +++ b/src/CodeGuard.Cli/Support/RuleAnalysisReportWriter.cs @@ -38,7 +38,7 @@ public static void WriteConsole(RuleAnalysisReport report, TextWriter writer) writer.WriteLine("Unreachable assertions:"); foreach (var issue in report.UnreachableRules) { - writer.WriteLine($" - {issue.RuleId}: '{issue.AssertionKind}' cannot apply to a '{issue.TargetKind}' target ({issue.SourceFile})"); + writer.WriteLine($" - {issue.RuleId}: '{issue.AssertionKind}' cannot apply to a '{issue.TargetKind}' target at {issue.Location} ({issue.SourceFile})"); } } diff --git a/src/CodeGuard.Configuration/Analysis/RuleSetAnalyzer.cs b/src/CodeGuard.Configuration/Analysis/RuleSetAnalyzer.cs index d2618c4..4f90fdd 100644 --- a/src/CodeGuard.Configuration/Analysis/RuleSetAnalyzer.cs +++ b/src/CodeGuard.Configuration/Analysis/RuleSetAnalyzer.cs @@ -2,11 +2,22 @@ using CodeGuard.Configuration.Loading; using CodeGuard.Configuration.Validation; using CodeGuard.RuleModel.Rules; +using System.Text.Json.Nodes; namespace CodeGuard.Configuration.Analysis; -/// One rule whose only known assertion can never fire against its target's candidate kind. -public sealed record UnreachableAssertionIssue(string RuleId, string SourceFile, string TargetKind, string AssertionKind); +/// +/// One assertion that can never fire against the candidate kind it is checked on. +/// is the selector kind whose candidates it was checked against (the rule's `target`, or the `selector` of +/// the quantifier it is nested in). is the path to the assertion inside the rule +/// document, e.g. assertions[0], assertions[0].assertions[1] or when.and[1]. +/// +public sealed record UnreachableAssertionIssue( + string RuleId, + string SourceFile, + string TargetKind, + string AssertionKind, + string Location); /// A set of rules whose target+assertions (or analyzer) body is structurally identical. public sealed record ExactDuplicateGroup(IReadOnlyList RuleIds, IReadOnlyList SourceFiles); @@ -105,13 +116,17 @@ private static bool CoversBothOutcomes(RuleDefinition rule) => rule.Tests.Any(t => t.Expect == TestExpectation.Pass) && rule.Tests.Any(t => t.Expect == TestExpectation.Fail); /// - /// Flags a top-level assertion that the target selector's candidate kind can never satisfy, using - /// the same / - /// metadata `rules discover` reports. Deliberately doesn't recurse into nested selectors/ - /// assertions (inside `must_all_match`/`must_any_match`/`must_none_match`/`must_have_count`) - - /// those aren't exposed by the parsed model (only `IAssertion.Kind` survives parsing; nested - /// selector/assertion objects stay in private constructor fields, the same limitation - /// `rules explain --format json` works around by reading the source document instead). An + /// Flags an assertion that the candidate kind it is checked against can never satisfy, using the + /// same / + /// metadata `rules discover` reports. Three places are checked: + /// + /// top-level assertions, against the target selector (from the parsed model); + /// assertions nested in a quantifier's `assertions:`, against that quantifier's `selector:`; + /// `when:` leaves, against the target selector. + /// + /// The last two need the raw document (): only `IAssertion.Kind` + /// survives parsing, so nested selector/assertion objects are otherwise unrecoverable - the same + /// reason `rules explain --format json` and the exact-duplicate check read the source document. An /// assertion with an empty `AppliesTo` (the quantifier/existence kinds, which evaluate a nested /// selector rather than the outer candidate) is never flagged - see that field's own doc comment. /// @@ -129,30 +144,129 @@ private static IReadOnlyList FindUnreachableAssertion continue; // analyzer-shaped rule - has no selector-driven target to check against. } - if (!selectorsByKind.TryGetValue(rule.Target.Kind, out var selectorDescriptor) - || selectorDescriptor.Produces is not { } produces) - { - continue; - } + var document = RuleFileLoader.ReadDocument(sourceFile).AsObject(); + var produces = selectorsByKind.TryGetValue(rule.Target.Kind, out var selectorDescriptor) + ? selectorDescriptor.Produces + : null; + var scanner = new NestedAssertionScanner( + rule.Id, sourceFile, selectorsByKind, assertionsByKind, issues); - foreach (var assertion in rule.Assertions) + for (var index = 0; index < rule.Assertions.Count; index++) { - if (!assertionsByKind.TryGetValue(assertion.Kind, out var assertionDescriptor) - || assertionDescriptor.AppliesTo.Count == 0) + var assertion = rule.Assertions[index]; + var location = $"assertions[{index}]"; + if (produces is { } candidate + && assertionsByKind.TryGetValue(assertion.Kind, out var assertionDescriptor) + && assertionDescriptor.AppliesTo.Count > 0 + && !assertionDescriptor.AppliesTo.Contains(candidate)) { - continue; + issues.Add(new UnreachableAssertionIssue( + rule.Id, sourceFile, rule.Target.Kind, assertion.Kind, location)); } - if (!assertionDescriptor.AppliesTo.Contains(produces)) - { - issues.Add(new UnreachableAssertionIssue(rule.Id, sourceFile, rule.Target.Kind, assertion.Kind)); - } + scanner.ScanNested(document["assertions"]?[index], location); } + + scanner.ScanCondition(document["when"], "when", rule.Target.Kind, produces); } return issues; } + /// + /// Walks one rule's raw document for the assertions the parsed model can't expose: those nested in a + /// quantifier's `assertions:` list, and `when:` leaves. Kinds missing from the catalog are skipped + /// silently - unknown kinds are rejected by schema validation and `rules validate`, not by this check. + /// + private sealed class NestedAssertionScanner( + string ruleId, + string sourceFile, + IReadOnlyDictionary selectorsByKind, + IReadOnlyDictionary assertionsByKind, + List issues) + { + /// Descends into a top-level assertion's nested selector/assertions, without re-checking the assertion itself. + public void ScanNested(JsonNode? assertionNode, string location) => + ScanAssertion(assertionNode, location, candidateKind: null, produces: null); + + /// Walks a `when:` tree - `and`/`or`/`not` combinators down to assertion leaves, each checked against the target. + public void ScanCondition(JsonNode? conditionNode, string location, string targetKind, CandidateKind? produces) + { + if (conditionNode is not JsonObject { Count: 1 } node) + { + return; + } + + var (key, value) = node.Single(); + switch (key) + { + case "and" or "or": + var children = value?.AsArray(); + for (var index = 0; children is not null && index < children.Count; index++) + { + ScanCondition(children[index], $"{location}.{key}[{index}]", targetKind, produces); + } + + break; + case "not": + ScanCondition(value, $"{location}.not", targetKind, produces); + break; + default: + ScanAssertion(node, location, targetKind, produces); + break; + } + } + + /// + /// Checks one `{ kind: params }` assertion against when given (a `when:` + /// leaf, or an assertion nested in a quantifier), then recurses into its own nested `selector:` and + /// `assertions:` if its descriptor declares an parameter. + /// + private void ScanAssertion(JsonNode? assertionNode, string location, string? candidateKind, CandidateKind? produces) + { + if (assertionNode is not JsonObject { Count: 1 } node) + { + return; + } + + var (kind, parameters) = node.Single(); + if (!assertionsByKind.TryGetValue(kind, out var descriptor)) + { + return; + } + + if (produces is { } candidate + && candidateKind is not null + && descriptor.AppliesTo.Count > 0 + && !descriptor.AppliesTo.Contains(candidate)) + { + issues.Add(new UnreachableAssertionIssue(ruleId, sourceFile, candidateKind, kind, location)); + } + + if (parameters is not JsonObject parameterObject + || !descriptor.Parameters.Any(p => p.Type == ParameterType.AssertionList)) + { + return; + } + + var nestedSelectorKind = parameterObject["selector"]?["kind"]?.GetValue(); + var nestedProduces = nestedSelectorKind is not null + && selectorsByKind.TryGetValue(nestedSelectorKind, out var selectorDescriptor) + ? selectorDescriptor.Produces + : null; + + var nested = parameterObject["assertions"]?.AsArray(); + for (var index = 0; nested is not null && index < nested.Count; index++) + { + ScanAssertion( + nested[index], + $"{location}.assertions[{index}]", + nestedSelectorKind, + nestedProduces); + } + } + } + /// /// Groups rules whose enforceable body () /// is structurally identical. Needs the raw source document, not the parsed model - diff --git a/tests/CodeGuard.Configuration.Tests/Analysis/RuleSetAnalyzerTests.cs b/tests/CodeGuard.Configuration.Tests/Analysis/RuleSetAnalyzerTests.cs index cd4de86..3ce7b2f 100644 --- a/tests/CodeGuard.Configuration.Tests/Analysis/RuleSetAnalyzerTests.cs +++ b/tests/CodeGuard.Configuration.Tests/Analysis/RuleSetAnalyzerTests.cs @@ -144,6 +144,170 @@ public void Analyze_QuantifierAssertion_IsNeverReportedAsUnreachable() Assert.Empty(report.UnreachableRules); } + [Theory] + [InlineData("must_all_match")] + [InlineData("must_any_match")] + [InlineData("must_none_match")] + public void Analyze_NestedAssertionThatCannotApplyToQuantifierSelector_IsReportedAsUnreachable(string quantifier) + { + // The nested selector yields Method candidates, but must_not_reference_project only accepts + // Project candidates - every method would fail it, so the nested assertion can never run. + WriteRuleFile("nested.yml", $$""" + id: DDD-ENTITY-001 + name: Some rule + target: + kind: class + namespace: "Contoso.Domain.Entities" + assertions: + - {{quantifier}}: + selector: + kind: method + name: "*" + assertions: + - must_not_reference_project: + name: "*Infrastructure*" + """); + + var report = Analyze(); + + var issue = Assert.Single(report.UnreachableRules); + Assert.Equal("DDD-ENTITY-001", issue.RuleId); + Assert.Equal("method", issue.TargetKind); + Assert.Equal("must_not_reference_project", issue.AssertionKind); + Assert.Equal("assertions[0].assertions[0]", issue.Location); + Assert.True(report.HasFindings); + } + + [Theory] + [InlineData("must_all_match")] + [InlineData("must_any_match")] + [InlineData("must_none_match")] + public void Analyze_NestedAssertionThatAppliesToQuantifierSelector_IsNotReported(string quantifier) + { + WriteRuleFile("nested-match.yml", $$""" + id: DDD-ENTITY-001 + name: Some rule + target: + kind: class + namespace: "Contoso.Domain.Entities" + assertions: + - {{quantifier}}: + selector: + kind: method + name: "*" + assertions: + - must_have_modifier: + modifier: static + """); + + var report = Analyze(); + + Assert.Empty(report.UnreachableRules); + } + + [Fact] + public void Analyze_QuantifierInsideQuantifier_ChecksInnerAssertionAgainstInnermostSelector() + { + // The outer quantifier's nested assertion is itself a quantifier, whose AppliesTo is empty and + // so isn't flagged - but its own nested assertion yields Method candidates, and is. + WriteRuleFile("nested-twice.yml", """ + id: DDD-ENTITY-001 + name: Some rule + target: + kind: class + namespace: "Contoso.Domain.Entities" + assertions: + - must_all_match: + selector: + kind: project + name: "*" + assertions: + - must_all_match: + selector: + kind: method + name: "*" + assertions: + - must_not_reference_project: + name: "*Infrastructure*" + """); + + var report = Analyze(); + + var issue = Assert.Single(report.UnreachableRules); + Assert.Equal("method", issue.TargetKind); + Assert.Equal("assertions[0].assertions[0].assertions[0]", issue.Location); + } + + [Theory] + [InlineData("and", "when.and[1]")] + [InlineData("or", "when.or[1]")] + [InlineData("not", "when.not")] + public void Analyze_WhenLeafThatCannotApplyToTarget_IsReportedAsUnreachable(string combinator, string expectedLocation) + { + // The target is a class (Type candidates); must_not_reference_project only accepts Project. + var whenNode = combinator == "not" + ? """ + not: + must_not_reference_project: + name: "*Infrastructure*" + """ + : $""" + {combinator}: + - must_inherit_from: + type: "Contoso.Domain.Entity" + - must_not_reference_project: + name: "*Infrastructure*" + """; + WriteRuleFile("when-mismatch.yml", $""" + id: DDD-ENTITY-001 + name: Some rule + target: + kind: class + namespace: "Contoso.Domain.Entities" + when: + {Indent(whenNode, 2)} + assertions: + - must_inherit_from: + type: "Contoso.Domain.Entity" + """); + + var report = Analyze(); + + var issue = Assert.Single(report.UnreachableRules); + Assert.Equal("class", issue.TargetKind); + Assert.Equal("must_not_reference_project", issue.AssertionKind); + Assert.Equal(expectedLocation, issue.Location); + } + + private static string Indent(string text, int spaces) => + string.Join("\n", text.Split('\n').Select(line => new string(' ', spaces) + line)); + + [Fact] + public void Analyze_WhenLeafThatAppliesToTarget_IsNotReported() + { + WriteRuleFile("when-match.yml", """ + id: DDD-ENTITY-001 + name: Some rule + target: + kind: class + namespace: "Contoso.Domain.Entities" + when: + and: + - must_inherit_from: + type: "Contoso.Domain.Entity" + - not: + must_have_modifier: + modifier: static + assertions: + - must_inherit_from: + type: "Contoso.Domain.Entity" + """); + + var report = Analyze(); + + Assert.Empty(report.UnreachableRules); + } + [Fact] public void Analyze_TwoRulesWithIdenticalTargetAndAssertions_AreGroupedAsExactDuplicates() { diff --git a/tests/CodeGuard.RuleFuzzing.Tests/Generation/RuleDocumentGenerator.cs b/tests/CodeGuard.RuleFuzzing.Tests/Generation/RuleDocumentGenerator.cs index a38c3f4..6cca427 100644 --- a/tests/CodeGuard.RuleFuzzing.Tests/Generation/RuleDocumentGenerator.cs +++ b/tests/CodeGuard.RuleFuzzing.Tests/Generation/RuleDocumentGenerator.cs @@ -28,14 +28,15 @@ public static Gen IncompatibleRule(CapabilityCatalog catalog) => /// /// Selector + a single must_all_match assertion whose *nested* selector/assertion pairing is /// guaranteed mismatched. The outer target/assertion pairing is never itself unreachable (quantifier - /// kinds have empty AppliesTo), so this exercises the documented gap where - /// RuleSetAnalyzer.FindUnreachableAssertions does not recurse into nested selectors. + /// kinds have empty AppliesTo), so this exercises that RuleSetAnalyzer.FindUnreachableAssertions + /// recurses into nested selectors and flags the mismatch. /// public static Gen NestedIncompatibleRule(CapabilityCatalog catalog) => SelectorGenerator.Any(catalog).SelectMany(outer => SelectorGenerator.Any(catalog).SelectMany(inner => (AssertionGenerator.GuaranteedMismatch(catalog, inner.Produces, 1) - ?? AssertionGenerator.Any(catalog, inner.Produces, PairingMode.Incompatible, 1)) + ?? throw new InvalidOperationException( + $"Catalog has no assertion that is guaranteed to mismatch '{inner.Produces}'.")) .Select(nestedAssertion => { var mustAllMatch = new JsonObject diff --git a/tests/CodeGuard.RuleFuzzing.Tests/UnreachableRuleCrossCheckFuzzTests.cs b/tests/CodeGuard.RuleFuzzing.Tests/UnreachableRuleCrossCheckFuzzTests.cs index 16623b1..ecb1933 100644 --- a/tests/CodeGuard.RuleFuzzing.Tests/UnreachableRuleCrossCheckFuzzTests.cs +++ b/tests/CodeGuard.RuleFuzzing.Tests/UnreachableRuleCrossCheckFuzzTests.cs @@ -13,9 +13,8 @@ namespace CodeGuard.RuleFuzzing.Tests; /// mismatch its selector () must be flagged by /// RuleSetAnalyzer.UnreachableRules, and must dynamically violate on every applicable candidate. /// A rule whose mismatch is nested inside a quantifier assertion -/// () exercises the documented gap where -/// RuleSetAnalyzer.FindUnreachableAssertions does not recurse into nested selectors - asserted -/// here as an explicit, monitored limitation rather than left an unmonitored blind spot. +/// () must be flagged too, since +/// RuleSetAnalyzer.FindUnreachableAssertions checks nested assertions against their nested selector. /// public sealed class UnreachableRuleCrossCheckFuzzTests(ITestOutputHelper output) { @@ -69,7 +68,7 @@ public void TopLevelMismatch_IsFlaggedByAnalyzer_AndAlwaysViolatesDynamically() } [Fact] - public void NestedMismatch_IsNotFlaggedByAnalyzer_AndNeverCrashesEvaluation() + public void NestedMismatch_IsFlaggedByAnalyzer_AndNeverCrashesEvaluation() { var gen = RuleDocumentGenerator.NestedIncompatibleRule(Harness.Catalog); var iterations = RuleFuzzOptions.Iterations("RULEFUZZ_ITERATIONS_NESTED_UNREACHABLE", 3000); @@ -87,11 +86,9 @@ public void NestedMismatch_IsNotFlaggedByAnalyzer_AndNeverCrashesEvaluation() } var report = Harness.AnalyzeSingleRule(rule, document); - - // Documented limitation, not a bug: FindUnreachableAssertions doesn't recurse into nested - // selectors inside must_all_match/must_any_match/must_none_match. If this ever starts - // failing, RuleSetAnalyzer has been taught to recurse and this test needs updating to match. - Assert.DoesNotContain(report.UnreachableRules, issue => issue.RuleId == rule.Id); + Assert.True( + report.UnreachableRules.Any(issue => issue.RuleId == rule.Id && issue.Location.StartsWith("assertions[0].assertions[", StringComparison.Ordinal)), + $"Expected '{rule.Id}' to have its nested mismatch flagged by RuleSetAnalyzer.\nDocument: {document.ToJsonString()}"); foreach (var (name, model) in RepositoryModelFixtures.All) {