From 893c43f6bfa07ea6b66af9714e0795b676197f46 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:47:44 +0100 Subject: [PATCH 01/60] chore: capture two trufflehog findings met while routing tool checks ahoy offers trufflehog as an optional dependency and asks a scan.deep question, but nothing in abcd runs trufflehog or reads scan.deep. Both were confirmed while finding the tool checks itd-63 reroutes. Refs: iss-2609261447331434 Refs: iss-2609261447395216 Assisted-by: Claude:claude-opus-5-5 --- ...rufflehog-as-an-optional-dependency-the-deps.md | 14 ++++++++++++++ ...asks-private-repo-trufflehog-present-confirm.md | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 .abcd/work/issues/open/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md create mode 100644 .abcd/work/issues/open/iss-2609261447395216-ahoy-install-asks-private-repo-trufflehog-present-confirm.md diff --git a/.abcd/work/issues/open/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md b/.abcd/work/issues/open/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md new file mode 100644 index 000000000..b2c7981ed --- /dev/null +++ b/.abcd/work/issues/open/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md @@ -0,0 +1,14 @@ +--- +schema_version: 1 +id: "iss-2609261447331434" +slug: "ahoy-offers-trufflehog-as-an-optional-dependency-the-deps" +severity: "minor" +category: "drift" +source: "agent-finding" +found_during: "autonomous run A resumed 2026-09-25" +origin: researcher-authored +production_mode: hand-written +found_at: "internal/core/ahoy/detect.go" +--- + +ahoy offers trufflehog as an optional dependency (the deps.trufflehog_missing gap, 'trufflehog enables deep secret scanning when scan.deep=true', fix hint 'brew install trufflehog') although nothing in abcd runs trufflehog: no adapter exists and scan.deep is read by no scanner. The gap asks a person to install a tool abcd never uses, which the explain-then-install mode (itd-63) would turn into an executed install of a program that does nothing for them. diff --git a/.abcd/work/issues/open/iss-2609261447395216-ahoy-install-asks-private-repo-trufflehog-present-confirm.md b/.abcd/work/issues/open/iss-2609261447395216-ahoy-install-asks-private-repo-trufflehog-present-confirm.md new file mode 100644 index 000000000..f938f816e --- /dev/null +++ b/.abcd/work/issues/open/iss-2609261447395216-ahoy-install-asks-private-repo-trufflehog-present-confirm.md @@ -0,0 +1,14 @@ +--- +schema_version: 1 +id: "iss-2609261447395216" +slug: "ahoy-install-asks-private-repo-trufflehog-present-confirm" +severity: "minor" +category: "drift" +source: "agent-finding" +found_during: "autonomous run A resumed 2026-09-25" +origin: researcher-authored +production_mode: hand-written +found_at: "internal/core/ahoy/apply.go" +--- + +ahoy install asks 'Private repo + trufflehog present — confirm deep secret scanning' and persists scan.deep in .abcd/config.json, but no scanner reads scan.deep and nothing runs trufflehog, so the answer changes nothing: a person told they enabled deep secret scanning has not. Either wire a trufflehog adapter behind the scanner seam or stop asking (and say the value is inert where it is already set). From 0289455ebd92d6766da29492d8bf94df2a97d887 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:50:28 +0100 Subject: [PATCH 02/60] feat(tools): the tool registry and the explain-then-install mode internal/core/tools is the mode itd-63 names: a curated registry of the tools abcd knows (gitleaks, gh), Explain(name, capability) for the plain-language explanation a verb gives when one is missing, and Install, which runs the registry's fixed argv only after the caller's Confirm returns yes and reports the verify result. Install is a trust boundary: an unknown tool is refused, CI never installs and is not asked, a nil confirmation is a no, the program must resolve on PATH outside the repository the verb ran from, and the step runs without a shell, with stdin closed, in its own process group killed through its handle on timeout. A no or a failure ends with the capability's standing ("continuing on the native secret scanner"). A tool the registry does not know gets a generic explanation, no step, and the capture that records the gap in abcd's own ledger. Assisted-by: Claude:claude-opus-5-5 --- internal/core/tools/explain.go | 148 ++++++++++++ internal/core/tools/install.go | 327 ++++++++++++++++++++++++++ internal/core/tools/registry.go | 174 ++++++++++++++ internal/core/tools/tools_test.go | 377 ++++++++++++++++++++++++++++++ 4 files changed, 1026 insertions(+) create mode 100644 internal/core/tools/explain.go create mode 100644 internal/core/tools/install.go create mode 100644 internal/core/tools/registry.go create mode 100644 internal/core/tools/tools_test.go diff --git a/internal/core/tools/explain.go b/internal/core/tools/explain.go new file mode 100644 index 000000000..8945b0527 --- /dev/null +++ b/internal/core/tools/explain.go @@ -0,0 +1,148 @@ +package tools + +import ( + "runtime" + "strings" +) + +// Explanation is what a verb says when it finds a tool missing (criterion 1): +// the tool, whether it is optional or required for this capability, what works +// without it, what it would do, and the exact install step, all from the +// registry. It is structured so each front door renders it its own way (the +// plugin page reads it from a verb's --json); Lines is the plain-text rendering +// every front door shares. +type Explanation struct { + Tool string `json:"tool"` + Known bool `json:"known"` + Capability Capability `json:"capability"` + CapabilityName string `json:"capability_name,omitempty"` + Requirement Requirement `json:"requirement,omitempty"` + What string `json:"what,omitempty"` + Homepage string `json:"homepage,omitempty"` + Does string `json:"does,omitempty"` + WithoutIt string `json:"without_it,omitempty"` + NativeDefault string `json:"native_default,omitempty"` + OnDecline string `json:"on_decline,omitempty"` + StepManager string `json:"step_manager,omitempty"` + // Step is the exact argv Install would run on this platform; empty when + // the registry names none (an unknown tool, or a platform without a step). + Step []string `json:"step,omitempty"` + Effects string `json:"effects,omitempty"` + Verify []string `json:"verify,omitempty"` + // RegistryGap is set for a tool the registry does not know: the gap is + // abcd's own, and this names the capture that records it. + RegistryGap string `json:"registry_gap,omitempty"` +} + +// Explain renders the explanation for name as capability uses it, on the +// running platform. +func Explain(name string, capability Capability) Explanation { + return explainFor(name, capability, runtime.GOOS) +} + +// explainFor is Explain with the platform named, so a test can reach every +// platform's step from any host. +func explainFor(name string, capability Capability, goos string) Explanation { + tool, ok := registry[name] + if !ok { + return unknown(name, capability) + } + e := Explanation{ + Tool: name, + Known: true, + Capability: capability, + What: tool.What, + Homepage: tool.Homepage, + Effects: tool.Effects, + Verify: append([]string(nil), tool.Verify...), + } + if use, ok := tool.Uses[capability]; ok { + e.CapabilityName = use.Capability + e.Requirement = use.Requirement + e.Does = use.Does + e.WithoutIt = use.WithoutIt + e.NativeDefault = use.NativeDefault + e.OnDecline = use.OnDecline + } else { + // A known tool asked about for a capability its entry does not name is + // a registry gap too, but a narrower one: what the tool is and how it is + // installed are still known, so only the use is generic. + e.CapabilityName = string(capability) + e.Requirement = Optional + e.Does = "abcd's registry does not say what this capability uses it for" + e.WithoutIt = "abcd's registry does not say" + e.OnDecline = "nothing was installed" + e.RegistryGap = gapCapture(name, capability) + } + if step, ok := tool.Install[goos]; ok { + e.StepManager = step.Manager + e.Step = append([]string(nil), step.Argv...) + } + return e +} + +// unknown is the generic explanation for a tool the registry does not know +// (criterion 3). It carries no step: abcd installs only what it can explain, +// and a step it does not hold is never composed from the tool's name. +func unknown(name string, capability Capability) Explanation { + return Explanation{ + Tool: name, + Capability: capability, + CapabilityName: string(capability), + What: "abcd's tool registry has no entry for it, so abcd cannot say what it is, why this capability " + + "wants it, or how to install it safely", + Does: "unknown to abcd", + WithoutIt: "unknown to abcd", + OnDecline: "nothing was installed", + RegistryGap: gapCapture(name, capability), + } +} + +// gapCapture is the capture that records a registry gap. The gap is abcd's own +// (its curated registry lacks an entry), so it is recorded in abcd's ledger by +// whoever meets it, and never written into the repository the verb ran in. +func gapCapture(name string, capability Capability) string { + return `abcd's tool registry has no entry for ` + name + ` as ` + string(capability) + + ` uses it; record the gap in abcd's own ledger with: abcd capture "the tool registry has no entry for ` + + name + ` (` + string(capability) + `)" --category future-work-seed` +} + +// StepText is the install step as a person would type it, or the reason there +// is none. +func (e Explanation) StepText() string { + if len(e.Step) == 0 { + if !e.Known { + return "none known to abcd (look the tool up before installing it)" + } + return "none known to abcd for this platform (see " + e.Homepage + ")" + } + return strings.Join(e.Step, " ") +} + +// Lines is the plain-text explanation, one line per part, the first naming the +// tool and whether it is optional or required. +func (e Explanation) Lines() []string { + if !e.Known { + return []string{ + e.Tool + " — " + e.What, + " install step: " + e.StepText(), + " registry gap: " + e.RegistryGap, + } + } + lines := []string{ + e.Tool + " — " + string(e.Requirement) + " for " + e.CapabilityName, + " what it is: " + e.What + " (" + e.Homepage + ")", + " what abcd uses it for: " + e.Does, + " without it: " + e.WithoutIt, + } + if len(e.Step) > 0 { + lines = append(lines, " install step ("+e.StepManager+"): "+e.StepText()) + } else { + lines = append(lines, " install step: "+e.StepText()) + } + lines = append(lines, " what the install does: "+e.Effects) + if e.RegistryGap != "" { + lines = append(lines, " registry gap: "+e.RegistryGap) + } + return lines +} diff --git a/internal/core/tools/install.go b/internal/core/tools/install.go new file mode 100644 index 000000000..3b47b4271 --- /dev/null +++ b/internal/core/tools/install.go @@ -0,0 +1,327 @@ +package tools + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "github.com/intentdriven/abcd/internal/fsutil" +) + +// Answer is a caller's reply to the install question: yes or no, and in words +// how the reply was reached (typed at a terminal, named with a flag, no +// terminal to ask at), which the result repeats so a no is never silent. +type Answer struct { + Yes bool + Why string +} + +// Confirm asks the person whether to run the install the explanation shows. +// The front door supplies it; a nil Confirm is a no. +type Confirm func(Explanation) Answer + +// Result reports one Install: what it ran and whether it worked, or why it ran +// nothing and what the capability continues on (criterion 2). +type Result struct { + Tool string `json:"tool"` + // Ran reports that the install step was executed. + Ran bool `json:"ran"` + // Argv is the step as run (argv[0] resolved), when Ran. + Argv []string `json:"argv,omitempty"` + // Installed reports that the step exited zero. + Installed bool `json:"installed"` + // Verified reports that the verify command exited zero afterwards. + Verified bool `json:"verified"` + // Declined reports that nobody said yes (a no, no confirmation, CI). + Declined bool `json:"declined"` + // Why is the reason nothing ran, or what failed. + Why string `json:"why,omitempty"` + // Output is the bounded tail of the step's output on a failure, or the + // verify command's first line on success. + Output string `json:"output,omitempty"` + // OnDecline is the capability's standing after a no or a failure. + OnDecline string `json:"on_decline,omitempty"` + step string + verify string +} + +// Summary is the one line a verb reports for the result. Every path that +// installed nothing ends with the capability's standing ("continuing on the +// native secret scanner"), so a no is loud rather than silent (spec scope 4). +func (r Result) Summary() string { + tail := "" + if r.OnDecline != "" { + tail = "; " + r.OnDecline + } + switch { + case !r.Ran: + return r.Tool + " not installed (" + r.Why + ")" + tail + case !r.Installed: + s := r.Tool + ": ran " + r.step + " — it failed (" + r.Why + ")" + if r.Output != "" { + s += ": " + r.Output + } + return s + tail + case !r.Verified: + s := r.Tool + ": ran " + r.step + " — it succeeded, but the verify " + r.verify + " failed (" + r.Why + ")" + if r.Output != "" { + s += ": " + r.Output + } + return s + tail + } + s := r.Tool + ": ran " + r.step + " — installed, and verified with " + r.verify + if r.Output != "" { + s += " (" + r.Output + ")" + } + return s +} + +// Installer runs confirmed install steps. Every process-touching seam is a +// field so the whole decision path is exercised in tests with no package +// manager; Default is the production wiring. +type Installer struct { + // LookPath resolves a bare program name on the operator's PATH. + LookPath func(string) (string, error) + // Run executes an argv whose argv[0] is already resolved, without a shell. + Run func(context.Context, []string) ([]byte, error) + // Getenv reads the environment (CI detection). + Getenv func(string) string + // GOOS selects the platform's step. + GOOS string + // Guard is the directory a resolved program must lie outside: the + // repository the verb was run from. A PATH entry pointing into it is + // repository content, which is trusted to choose nothing that runs. + Guard string +} + +// Default is the production installer for a verb run from guard (its +// repository root, or its working directory outside one). +func Default(guard string) *Installer { + return &Installer{LookPath: exec.LookPath, Run: runArgv, Getenv: os.Getenv, GOOS: runtime.GOOS, Guard: guard} +} + +// Install explains name as capability uses it, asks confirm, and on a yes runs +// the registry's step for this platform and then its verify command. It is the +// package-level spelling of Default(guard).Install. +func Install(name string, capability Capability, confirm Confirm, guard string) Result { + return Default(guard).Install(name, capability, confirm) +} + +const ( + // installTimeout bounds a package manager run; a timed-out step's whole + // process group is killed through its own handle. + installTimeout = 15 * time.Minute + // verifyTimeout bounds the verify command. + verifyTimeout = 30 * time.Second + // maxOutput bounds what is kept of a step's output. + maxOutput = 64 * 1024 + // tailLines is how much of a failure's output the summary carries. + tailLines = 6 +) + +// Install is the trust boundary. In order, and each refusal runs nothing: +// +// 1. An unknown tool is refused: nothing is composed from a name. +// 2. A platform without a step is refused. +// 3. CI never installs, and is not asked. +// 4. The caller's Confirm must return yes; nil is a no. +// 5. The step's program must resolve on PATH to an absolute path outside the +// guarded tree, lexically and after symlinks. +// +// The step then runs as an argv (no shell), with stdin closed, in its own +// process group, bounded in time; the verify command follows under the same +// rules. +func (in *Installer) Install(name string, capability Capability, confirm Confirm) Result { + e := explainFor(name, capability, in.GOOS) + r := Result{Tool: name, OnDecline: e.OnDecline} + if !e.Known { + r.Declined = true + r.Why = "abcd's tool registry has no entry for " + name + ", so it holds no install step it trusts; " + + "it knows " + strings.Join(Names(), ", ") + return r + } + if len(e.Step) == 0 { + r.Why = "abcd's registry holds no install step for " + in.GOOS + "; see " + e.Homepage + return r + } + r.step = strings.Join(e.Step, " ") + r.verify = strings.Join(e.Verify, " ") + if ci := in.Getenv("CI"); ci != "" { + r.Declined = true + r.Why = "CI is set: abcd never installs a tool in CI" + return r + } + if confirm == nil { + r.Declined = true + r.Why = "no one was asked" + return r + } + ans := confirm(e) + if !ans.Yes { + r.Declined = true + r.Why = ans.Why + if r.Why == "" { + r.Why = "the answer was no" + } + return r + } + prog, err := in.admit(e.Step[0]) + if err != nil { + if errors.Is(err, errNotFound) { + r.Why = e.StepManager + " (" + e.Step[0] + ") is not on PATH, so the step cannot run; install " + + e.StepManager + " first, or install " + name + " by the route " + e.Homepage + " gives" + } else { + r.Why = err.Error() + } + return r + } + argv := append([]string{prog}, e.Step[1:]...) + r.Ran = true + r.Argv = argv + ctx, cancel := context.WithTimeout(context.Background(), installTimeout) + out, err := in.Run(ctx, argv) + cancel() + if err != nil { + r.Why = err.Error() + r.Output = tail(out) + return r + } + r.Installed = true + vprog, err := in.admit(e.Verify[0]) + if err != nil { + r.Why = err.Error() + return r + } + ctx, cancel = context.WithTimeout(context.Background(), verifyTimeout) + vout, err := in.Run(ctx, append([]string{vprog}, e.Verify[1:]...)) + cancel() + if err != nil { + r.Why = err.Error() + r.Output = tail(vout) + return r + } + r.Verified = true + r.OnDecline = "" + r.Output = firstLine(vout) + return r +} + +var errNotFound = errors.New("not found on PATH") + +// admit resolves a bare program name on PATH and refuses a result inside the +// guarded tree. PATH is the operator's environment, which abcd trusts to locate +// git and gh everywhere else; what a checkout CAN reach is a PATH entry that +// points into it, so the result is judged as the gitleaks adapter judges a +// binary: absolute, and outside the guarded tree both lexically and after +// symlink resolution. An empty guard is refused rather than trusted. +func (in *Installer) admit(name string) (string, error) { + p, err := in.LookPath(name) + if err != nil { + return "", fmt.Errorf("%s: %w", name, errNotFound) + } + if !filepath.IsAbs(p) { + return "", fmt.Errorf("%s resolves to %q, which is not an absolute path; nothing was run", name, p) + } + if in.Guard == "" || !filepath.IsAbs(in.Guard) { + return "", fmt.Errorf("%s cannot be judged without an absolute directory to hold it outside; nothing was run", name) + } + resolved, err := filepath.EvalSymlinks(p) + if err != nil { + return "", fmt.Errorf("%s resolves to %q, which does not resolve; nothing was run", name, p) + } + guards := []string{filepath.Clean(in.Guard)} + if g, err := filepath.EvalSymlinks(in.Guard); err == nil { + guards = append(guards, g) + } + fold := fsutil.CaseFoldingFS() + for _, g := range guards { + for _, c := range []string{filepath.Clean(p), resolved} { + if fsutil.PathWithin(c, g, fold) { + return "", fmt.Errorf("%s resolves to %q, inside the repository this verb ran from; "+ + "a program there is repository content and is never run as an install step", name, p) + } + } + } + return resolved, nil +} + +// runArgv is the production runner: the argv is executed directly (no shell), +// stdin is the null device so a package manager cannot stop to ask, the working +// directory is the temporary directory rather than the repository, and the +// child leads its own process group so a timeout kills everything it started +// through this handle and nothing else. +func runArgv(ctx context.Context, argv []string) ([]byte, error) { + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = os.TempDir() + cmd.Stdin = nil + var buf bytes.Buffer + w := &capped{w: &buf, remaining: maxOutput} + cmd.Stdout = w + cmd.Stderr = w + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + return nil, err + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + return buf.Bytes(), err + case <-ctx.Done(): + // The group is the one this child leads (Setpgid), addressed through + // the pid this handle holds: never a pattern, never another process. + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + <-done + return buf.Bytes(), fmt.Errorf("did not finish within the time allowed (%w)", ctx.Err()) + } +} + +// capped keeps at most remaining bytes and drops the rest without failing the +// writer, so a chatty package manager is never killed for its verbosity. +type capped struct { + w *bytes.Buffer + remaining int +} + +func (c *capped) Write(p []byte) (int, error) { + n := len(p) + if c.remaining > 0 { + k := n + if k > c.remaining { + k = c.remaining + } + c.w.Write(p[:k]) + c.remaining -= k + } + return n, nil +} + +// tail is the last few non-blank lines of out, joined on " | ". +func tail(out []byte) string { + var keep []string + for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if l = strings.TrimSpace(l); l != "" { + keep = append(keep, l) + } + } + if len(keep) > tailLines { + keep = keep[len(keep)-tailLines:] + } + return strings.Join(keep, " | ") +} + +func firstLine(out []byte) string { + s := strings.TrimSpace(string(out)) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + return s +} diff --git a/internal/core/tools/registry.go b/internal/core/tools/registry.go new file mode 100644 index 000000000..e48c83caf --- /dev/null +++ b/internal/core/tools/registry.go @@ -0,0 +1,174 @@ +// Package tools is abcd's explain-then-install mode (itd-63): the curated +// registry of the external tools abcd knows, the plain-language explanation a +// verb gives when it finds one missing, and the one place a confirmed install +// step is run. +// +// It is a MODE other verbs call, never a surface of its own (the product +// thinker's decision 3, 2026-09-21): a verb that finds a tool missing asks +// Explain for what to say and, where it offers the install, hands Install a +// Confirm function its front door supplies. The package has no transport +// knowledge — it never reads a terminal and never writes to stdout — so the CLI +// asks on the terminal, the plugin page asks through the host's question tool, +// and this code is the same under both (criterion 5). +// +// The trust boundary is Install. What it runs is fixed data in this file, +// compiled into the binary: an argv per platform, never composed from a flag, a +// repository file or an environment value, never a shell string. It runs only +// after the caller's Confirm returns yes, never in CI, and never a program that +// resolves inside the repository it was asked from. See install.go. +package tools + +import "sort" + +// Requirement is whether a capability needs a tool or merely works better with +// it (adr-22: most dependencies are optional adapters over a native default). +type Requirement string + +const ( + // Optional: the capability runs on its native default without the tool. + Optional Requirement = "optional" + // Required: the capability cannot run without the tool. + Required Requirement = "required" +) + +// Capability names what a verb is doing when it finds a tool missing. The same +// tool is optional for one capability and required for another, so an +// explanation is always for a (tool, capability) pair. +type Capability string + +const ( + // TranscriptScan is the secret scan of captured session transcripts in a + // repository that has NOT armed gitleaks: the native scanner covers it. + TranscriptScan Capability = "transcript-scan" + // TranscriptScanArmed is the same scan in a repository that armed gitleaks + // in .abcd/config/gitleaks.json: the history store refuses to store a + // transcript with less coverage than the repository asked for. + TranscriptScanArmed Capability = "transcript-scan-armed" + // RemoteSettings is `ahoy remote`: reading and changing the repository's + // GitHub secret-scanning settings, which abcd does only through gh. + RemoteSettings Capability = "remote-settings" +) + +// Use is what one capability does with a tool, in the words a person reads. +type Use struct { + // Capability is the capability's human name. + Capability string + // Requirement is optional or required, for this capability. + Requirement Requirement + // Does is what abcd uses the tool for here. + Does string + // WithoutIt is what works without the tool (for an optional use, the + // native default; for a required one, what fails and the way back). + WithoutIt string + // NativeDefault is what the capability continues on after a no; empty when + // there is none. + NativeDefault string + // OnDecline is the loud line's tail after a no or a failed install: + // "continuing on " for an optional use, the standing + // refusal for a required one. + OnDecline string +} + +// Step is one platform's install step: the package manager it uses, by name, +// and the exact argv run. Argv[0] is a bare program name resolved on PATH at +// run time; the rest are literal arguments. +type Step struct { + Manager string + Argv []string +} + +// Tool is one registry entry. +type Tool struct { + Name string + What string // what the tool is, plainly + Homepage string // where to read what it is + Uses map[Capability]Use + Install map[string]Step // keyed by runtime.GOOS + // Effects is what the install does to the machine, and what it does not. + Effects string + // Verify is the argv that proves the tool runs once installed. + Verify []string +} + +// homebrew builds the Homebrew step for a formula. Homebrew is the one package +// manager with a single fixed command on both supported platforms (macOS, and +// Linux through Homebrew on Linux); a machine without it is told so, never +// offered a step composed for some other manager. +func homebrew(formula string) Step { + return Step{Manager: "Homebrew", Argv: []string{"brew", "install", formula}} +} + +// registry is the curated set (the product thinker's decision 2). An entry is +// added by a change to this file, never at run time: a gap met at run time is +// named as abcd's own and captured, not composed (explain.go). +var registry = map[string]Tool{ + "gitleaks": { + Name: "gitleaks", + What: "an open-source secret scanner: it reads text and reports the credentials it recognises " + + "(API keys, access tokens, private keys)", + Homepage: "https://github.com/gitleaks/gitleaks", + Uses: map[Capability]Use{ + TranscriptScan: { + Capability: "the secret scan of captured session transcripts", + Requirement: Optional, + Does: "when a repository opts in with .abcd/config/gitleaks.json, abcd runs gitleaks over each " + + "transcript before storing it and masks what it finds, on top of the native scanner; it reaches " + + "labelled, high-entropy keys in prose that the native patterns miss", + WithoutIt: "the native secret scanner built into abcd still scans and masks every transcript and " + + "every launch payload", + NativeDefault: "the native secret scanner", + OnDecline: "continuing on the native secret scanner", + }, + TranscriptScanArmed: { + Capability: "the secret scan of captured session transcripts, which this repository armed in .abcd/config/gitleaks.json", + Requirement: Required, + Does: "abcd runs gitleaks over each transcript before storing it and masks what it finds, on top of " + + "the native scanner, because this repository asked for that coverage", + WithoutIt: "nothing is stored: abcd refuses to store this repository's transcripts with less coverage " + + "than the repository asked for; setting enabled to false in .abcd/config/gitleaks.json returns it " + + "to the native secret scanner", + OnDecline: "transcript capture stays refused for this repository until gitleaks is installed or " + + ".abcd/config/gitleaks.json sets enabled to false", + }, + }, + Install: map[string]Step{"darwin": homebrew("gitleaks"), "linux": homebrew("gitleaks")}, + Effects: "Homebrew downloads the gitleaks program and puts it on your PATH. It changes no repository, " + + "starts no background service, and sends nothing anywhere; abcd runs it only in a repository that opts in.", + Verify: []string{"gitleaks", "version"}, + }, + "gh": { + Name: "gh", + What: "GitHub's own command-line program, signed in as you", + Homepage: "https://cli.github.com", + Uses: map[Capability]Use{ + RemoteSettings: { + Capability: "reading and changing this repository's GitHub secret-scanning settings (ahoy remote)", + Requirement: Required, + Does: "abcd speaks to GitHub through gh, so a remote change is made by your own signed-in identity " + + "and abcd never holds a token", + WithoutIt: "ahoy remote cannot read or change the settings; nothing else in abcd needs gh", + OnDecline: "the remote settings stay unread and unchanged", + }, + }, + Install: map[string]Step{"darwin": homebrew("gh"), "linux": homebrew("gh")}, + Effects: "Homebrew downloads the gh program and puts it on your PATH. It does not sign you in: run " + + "gh auth login afterwards, and abcd never sees the token.", + Verify: []string{"gh", "--version"}, + }, +} + +// Names returns the registered tool names, sorted. +func Names() []string { + out := make([]string, 0, len(registry)) + for n := range registry { + out = append(out, n) + } + sort.Strings(out) + return out +} + +// Known reports whether the registry holds an entry for name. +func Known(name string) bool { + _, ok := registry[name] + return ok +} diff --git a/internal/core/tools/tools_test.go b/internal/core/tools/tools_test.go new file mode 100644 index 000000000..53c0dc4a8 --- /dev/null +++ b/internal/core/tools/tools_test.go @@ -0,0 +1,377 @@ +package tools + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// TestExplainNamesEveryPartTheCriterionAsksFor is itd-63 criterion 1: a named +// gap states the tool's name, whether it is optional or required for THIS +// capability, what works without it, what the tool would do, and the exact +// install step, all from the registry entry. +func TestExplainNamesEveryPartTheCriterionAsksFor(t *testing.T) { + e := explainFor("gitleaks", TranscriptScan, "darwin") + if !e.Known { + t.Fatal("gitleaks is not in the registry") + } + if e.Requirement != Optional { + t.Errorf("requirement = %q, want optional for the plain transcript scan", e.Requirement) + } + if got := strings.Join(e.Step, " "); got != "brew install gitleaks" { + t.Errorf("step = %q, want the exact argv brew install gitleaks", got) + } + text := strings.Join(e.Lines(), "\n") + for _, want := range []string{ + "gitleaks", + "optional", + "without it:", + "native secret scanner", + "what abcd uses it for:", + "install step (Homebrew): brew install gitleaks", + "what the install does:", + "https://github.com/gitleaks/gitleaks", + } { + if !strings.Contains(text, want) { + t.Errorf("explanation lacks %q:\n%s", want, text) + } + } +} + +// TestExplainIsPerCapability: the same tool is optional for one capability and +// required for another, and the explanation says which, with what a no means. +func TestExplainIsPerCapability(t *testing.T) { + e := explainFor("gitleaks", TranscriptScanArmed, "linux") + if e.Requirement != Required { + t.Fatalf("requirement = %q, want required where the repository armed gitleaks", e.Requirement) + } + if !strings.Contains(e.WithoutIt, "enabled") { + t.Errorf("an armed repository's without-it must name the opt-out that returns it to the native scanner: %q", e.WithoutIt) + } + gh := explainFor("gh", RemoteSettings, "darwin") + if gh.Requirement != Required || !gh.Known { + t.Fatalf("gh for the remote settings: %+v", gh) + } + if !strings.Contains(gh.Effects, "gh auth login") { + t.Errorf("gh's install effects must say it does not sign the person in: %q", gh.Effects) + } +} + +// TestUnknownToolGetsGenericTextAndARegistryGap is criterion 3: an unknown tool +// gets the generic explanation, no install step abcd would run, and the gap is +// named as abcd's own with the capture that records it. +func TestUnknownToolGetsGenericTextAndARegistryGap(t *testing.T) { + e := explainFor("frobnicate", TranscriptScan, "darwin") + if e.Known { + t.Fatal("an unregistered tool reads as known") + } + if len(e.Step) != 0 { + t.Fatalf("an unregistered tool carries a step abcd would run: %v", e.Step) + } + if e.RegistryGap == "" || !strings.Contains(e.RegistryGap, "abcd capture") || !strings.Contains(e.RegistryGap, "frobnicate") { + t.Fatalf("registry gap does not carry the capture naming the tool: %q", e.RegistryGap) + } + text := strings.Join(e.Lines(), "\n") + for _, want := range []string{"frobnicate", "no entry", "install step: none"} { + if !strings.Contains(text, want) { + t.Errorf("generic explanation lacks %q:\n%s", want, text) + } + } +} + +// TestUnknownPlatformHasNoStep: a platform the entry names no step for shows +// none rather than borrowing another platform's. +func TestUnknownPlatformHasNoStep(t *testing.T) { + e := explainFor("gitleaks", TranscriptScan, "plan9") + if len(e.Step) != 0 { + t.Fatalf("plan9 borrowed a step: %v", e.Step) + } + if !strings.Contains(strings.Join(e.Lines(), "\n"), "install step: none") { + t.Fatalf("no-step platform does not say so:\n%s", strings.Join(e.Lines(), "\n")) + } +} + +// TestEveryRegistryEntryIsComplete keeps the registry honest: every entry and +// every use carries each part the explanation renders, and every step is a +// fixed argv whose program is a bare name (resolved on PATH at run time), +// never a shell. +func TestEveryRegistryEntryIsComplete(t *testing.T) { + if len(Names()) == 0 { + t.Fatal("empty registry") + } + for _, name := range Names() { + tool := registry[name] + if tool.Name != name || tool.What == "" || tool.Homepage == "" || tool.Effects == "" || len(tool.Verify) == 0 { + t.Errorf("%s: incomplete entry %+v", name, tool) + } + if tool.Verify[0] != name { + t.Errorf("%s: verify runs %q, not the tool itself", name, tool.Verify[0]) + } + if len(tool.Uses) == 0 { + t.Errorf("%s: no capability uses it", name) + } + for c, u := range tool.Uses { + if u.Capability == "" || u.Does == "" || u.WithoutIt == "" || u.OnDecline == "" { + t.Errorf("%s/%s: incomplete use %+v", name, c, u) + } + if u.Requirement != Optional && u.Requirement != Required { + t.Errorf("%s/%s: requirement %q", name, c, u.Requirement) + } + } + for goos, s := range tool.Install { + if len(s.Argv) < 2 || s.Manager == "" { + t.Errorf("%s/%s: incomplete step %+v", name, goos, s) + } + for _, a := range s.Argv { + if strings.ContainsAny(a, " ;|&$`<>\\\"'") { + t.Errorf("%s/%s: argv element %q carries shell syntax", name, goos, a) + } + } + if s.Argv[0] == "sh" || s.Argv[0] == "bash" || s.Argv[0] == "sudo" || strings.Contains(s.Argv[0], "/") { + t.Errorf("%s/%s: program %q is a shell, sudo or a path", name, goos, s.Argv[0]) + } + } + } +} + +// --- Install ----------------------------------------------------------------- + +// fakeExec records every argv it is handed and answers from a table. +type fakeExec struct { + calls [][]string + fail map[string]error // keyed by the program's base name + out map[string]string +} + +func (f *fakeExec) run(_ context.Context, argv []string) ([]byte, error) { + f.calls = append(f.calls, append([]string(nil), argv...)) + base := filepath.Base(argv[0]) + return []byte(f.out[base]), f.fail[base] +} + +// binDir plants executables named for each program outside any repository, +// so the lookup finds them and admission passes. +func binDir(t *testing.T, names ...string) string { + t.Helper() + dir := t.TempDir() + for _, n := range names { + if err := os.WriteFile(filepath.Join(dir, n), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + } + return dir +} + +func testInstaller(t *testing.T, fx *fakeExec, bin string, env map[string]string) *Installer { + t.Helper() + return &Installer{ + LookPath: func(name string) (string, error) { + p := filepath.Join(bin, name) + if _, err := os.Stat(p); err != nil { + return "", errors.New("not found") + } + return p, nil + }, + Run: fx.run, + Getenv: func(k string) string { return env[k] }, + GOOS: "darwin", + Guard: t.TempDir(), + } +} + +func yes(Explanation) Answer { return Answer{Yes: true, Why: "typed yes"} } +func no(Explanation) Answer { return Answer{Why: "typed no"} } + +// TestInstallRunsTheRegistryStepOnlyOnYesAndVerifies is criterion 2's yes half: +// the fixed argv runs, then the verify command, and the result says both. +func TestInstallRunsTheRegistryStepOnlyOnYesAndVerifies(t *testing.T) { + fx := &fakeExec{out: map[string]string{"gitleaks": "8.18.0\n"}} + bin := binDir(t, "brew", "gitleaks") + in := testInstaller(t, fx, bin, nil) + var asked Explanation + r := in.Install("gitleaks", TranscriptScan, func(e Explanation) Answer { asked = e; return yes(e) }) + if asked.Tool != "gitleaks" || len(asked.Step) == 0 { + t.Fatalf("the confirmation was not handed the explanation: %+v", asked) + } + if !r.Ran || !r.Installed || !r.Verified || r.Declined { + t.Fatalf("result = %+v, want ran, installed, verified", r) + } + if len(fx.calls) != 2 { + t.Fatalf("calls = %v, want the step then the verify", fx.calls) + } + // What runs is the symlink-resolved path, so what was judged and what is + // executed are the same bytes. + bin, _ = filepath.EvalSymlinks(bin) + if got := fx.calls[0]; got[0] != filepath.Join(bin, "brew") || strings.Join(got[1:], " ") != "install gitleaks" { + t.Errorf("step argv = %v", got) + } + if got := fx.calls[1]; got[0] != filepath.Join(bin, "gitleaks") || strings.Join(got[1:], " ") != "version" { + t.Errorf("verify argv = %v", got) + } + s := r.Summary() + if !strings.Contains(s, "ran brew install gitleaks") || !strings.Contains(s, "verified") { + t.Errorf("summary does not report what ran and whether it worked: %q", s) + } +} + +// TestInstallOnNoRunsNothingAndSaysContinuing is criterion 2's no half and the +// spec's loud staging: nothing runs, and the result carries the +// "continuing on " line. +func TestInstallOnNoRunsNothingAndSaysContinuing(t *testing.T) { + fx := &fakeExec{} + in := testInstaller(t, fx, binDir(t, "brew"), nil) + r := in.Install("gitleaks", TranscriptScan, no) + if r.Ran || !r.Declined || len(fx.calls) != 0 { + t.Fatalf("a no ran something: %+v calls=%v", r, fx.calls) + } + want := "gitleaks not installed (typed no); continuing on the native secret scanner" + if r.Summary() != want { + t.Fatalf("summary = %q, want %q", r.Summary(), want) + } +} + +// TestInstallNeverRunsWithoutAConfirmation: a nil confirmation is a no. +func TestInstallNeverRunsWithoutAConfirmation(t *testing.T) { + fx := &fakeExec{} + r := testInstaller(t, fx, binDir(t, "brew"), nil).Install("gitleaks", TranscriptScan, nil) + if r.Ran || len(fx.calls) != 0 || !r.Declined { + t.Fatalf("installed without a confirmation: %+v", r) + } +} + +// TestInstallNeverRunsInCI: CI never installs, whatever the confirmation says, +// and the confirmation is not even asked. +func TestInstallNeverRunsInCI(t *testing.T) { + fx := &fakeExec{} + asked := false + in := testInstaller(t, fx, binDir(t, "brew"), map[string]string{"CI": "true"}) + r := in.Install("gitleaks", TranscriptScan, func(e Explanation) Answer { asked = true; return yes(e) }) + if r.Ran || len(fx.calls) != 0 || asked { + t.Fatalf("CI installed or asked: %+v asked=%v", r, asked) + } + if !strings.Contains(r.Summary(), "CI") || !strings.Contains(r.Summary(), "continuing on") { + t.Fatalf("CI refusal is not said: %q", r.Summary()) + } +} + +// TestInstallRefusesAnUnknownTool: nothing composed, nothing run, and the +// refusal names the tools the registry does know. +func TestInstallRefusesAnUnknownTool(t *testing.T) { + fx := &fakeExec{} + asked := false + r := testInstaller(t, fx, binDir(t, "brew"), nil).Install("frobnicate", TranscriptScan, func(e Explanation) Answer { asked = true; return yes(e) }) + if r.Ran || len(fx.calls) != 0 || asked { + t.Fatalf("an unknown tool ran or was asked about: %+v", r) + } + for _, n := range Names() { + if !strings.Contains(r.Why, n) { + t.Errorf("refusal does not name known tool %q: %q", n, r.Why) + } + } +} + +// TestInstallSaysWhenThePackageManagerIsMissing: a yes with no Homebrew runs +// nothing and says what is missing. +func TestInstallSaysWhenThePackageManagerIsMissing(t *testing.T) { + fx := &fakeExec{} + r := testInstaller(t, fx, binDir(t), nil).Install("gitleaks", TranscriptScan, yes) + if r.Ran || len(fx.calls) != 0 { + t.Fatalf("ran without a package manager: %+v", r) + } + if !strings.Contains(r.Why, "Homebrew") || !strings.Contains(r.Summary(), "continuing on") { + t.Fatalf("missing manager not said: %q", r.Summary()) + } +} + +// TestInstallRefusesAProgramInsideTheGuardedTree: a PATH entry that resolves +// the package manager into the repository is repository content, and never +// runs. +func TestInstallRefusesAProgramInsideTheGuardedTree(t *testing.T) { + fx := &fakeExec{} + repo := t.TempDir() + bin := filepath.Join(repo, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bin, "brew"), []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + in := testInstaller(t, fx, bin, nil) + in.Guard = repo + r := in.Install("gitleaks", TranscriptScan, yes) + if r.Ran || len(fx.calls) != 0 { + t.Fatalf("ran a package manager from inside the repository: %+v", r) + } + if !strings.Contains(r.Why, "inside") { + t.Fatalf("refusal does not say why: %q", r.Why) + } +} + +// TestInstallReportsAFailedStepAndAFailedVerify: both failures are reported, +// never a success. +func TestInstallReportsAFailedStepAndAFailedVerify(t *testing.T) { + fx := &fakeExec{fail: map[string]error{"brew": errors.New("exit status 1")}, out: map[string]string{"brew": "Error: no bottle\n"}} + r := testInstaller(t, fx, binDir(t, "brew", "gitleaks"), nil).Install("gitleaks", TranscriptScan, yes) + if !r.Ran || r.Installed || r.Verified { + t.Fatalf("a failed step reads as installed: %+v", r) + } + if !strings.Contains(r.Summary(), "failed") || !strings.Contains(r.Summary(), "no bottle") || !strings.Contains(r.Summary(), "continuing on") { + t.Fatalf("failed step summary: %q", r.Summary()) + } + + fx = &fakeExec{fail: map[string]error{"gitleaks": errors.New("exit status 2")}} + r = testInstaller(t, fx, binDir(t, "brew", "gitleaks"), nil).Install("gitleaks", TranscriptScan, yes) + if !r.Installed || r.Verified { + t.Fatalf("a failed verify reads as verified: %+v", r) + } + if !strings.Contains(r.Summary(), "verify") || !strings.Contains(r.Summary(), "failed") { + t.Fatalf("failed verify summary: %q", r.Summary()) + } +} + +// TestRunArgvExecutesWithoutAShell drives the production runner: the argv is +// handed to the program as-is, so a metacharacter reaches it as a literal byte +// rather than being interpreted. +func TestRunArgvExecutesWithoutAShell(t *testing.T) { + dir := t.TempDir() + out := filepath.Join(dir, "argv") + script := filepath.Join(dir, "echoargs") + body := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + out + "\n" + if err := os.WriteFile(script, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + if _, err := runArgv(context.Background(), []string{script, "a;b", "$(x)"}); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if string(got) != "a;b\n$(x)\n" { + t.Fatalf("argv reached the program as %q", got) + } +} + +// TestRunArgvKillsItsOwnGroupOnTimeout: a step that outlives its bound is +// stopped with everything it started, through the handle, promptly. +func TestRunArgvKillsItsOwnGroupOnTimeout(t *testing.T) { + dir := t.TempDir() + script := filepath.Join(dir, "hang") + if err := os.WriteFile(script, []byte("#!/bin/sh\nsleep 30 &\nsleep 30\n"), 0o755); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + start := time.Now() + _, err := runArgv(ctx, []string{script}) + if err == nil || !strings.Contains(err.Error(), "did not finish") { + t.Fatalf("err = %v, want the timeout named", err) + } + if d := time.Since(start); d > 10*time.Second { + t.Fatalf("runArgv waited %s: the group was not killed", d) + } +} From 1e36dec112f264300f1b047564c7b6087648a0ff Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:53:45 +0100 Subject: [PATCH 03/60] feat(record): the canonical term-overlap primitive for matching at filing internal/core/record/match scores how much of a new text another record already holds, weighted by how rare each shared term is across the candidates, and proposes `duplicates` (high both ways) or `refines` (high one way: the new text is the narrower) above a threshold, listing the best below it as near misses with their scores. It declares itself a lexical heuristic on every outcome, never proposes a reversal or a supersession (the itd-84 discipline keeps those advisory and human), and files a text below a declared minimum of terms without comparing it. The spec (spc-2609212141417782) names the embark ranking's overlap function as the one to move here. That ranking is unbuilt (scope 7 of the reflect spec, still open), so there is no copy to move: this is the primitive that ranking calls when it lands. It is a subpackage of record rather than record itself because record imports capture and intent, which both call it; a leaf is what avoids the cycle. The 0.6 default was read off this repository's ledger: each of the 449 open issues matched against the other 1,660 issues and intents, 14 cleared it, most of them a real double or a narrower follow-up. Assisted-by: Claude:claude-opus-5-5 --- internal/core/record/match/match.go | 314 +++++++++++++++++++++++ internal/core/record/match/match_test.go | 168 ++++++++++++ 2 files changed, 482 insertions(+) create mode 100644 internal/core/record/match/match.go create mode 100644 internal/core/record/match/match_test.go diff --git a/internal/core/record/match/match.go b/internal/core/record/match/match.go new file mode 100644 index 000000000..6b076d4ba --- /dev/null +++ b/internal/core/record/match/match.go @@ -0,0 +1,314 @@ +// Package match is the canonical term-overlap primitive: it scores how much of +// a new text another record already holds, by the words the two share, and +// classifies a likely double as `duplicates` (near-identical) or `refines` +// (narrower). It is a lexical HEURISTIC and says so on every result it returns +// (Heuristic): no model is consulted, nothing semantic is claimed, and a match +// is a proposal a person confirms or removes (itd-2609212137116617). +// +// It is the one home of the overlap score (the one-canonical-primitive rule): +// the filing-time match of `capture` and of the quoted-text `intent` create +// call it, and any later ranking by shared terms (the embark ranking the +// reflect spec names) calls it rather than growing a copy. It sits below the +// record dispatcher rather than inside it because the dispatcher reads the +// capture and intent stores, and both of those call this package; a leaf +// package is what lets every caller reach it without an import cycle. +// +// The package reads nothing, writes nothing and never prints. Its callers +// gather the candidate texts under their own locks and write the links. +package match + +import ( + "math" + "sort" + "strings" + "unicode" +) + +// Heuristic names the method on every outcome, so a surface never renders a +// score as more than it is. +const Heuristic = "lexical term overlap, weighted by how rare each term is across the candidates; a heuristic, not a judgement" + +// Relation is the typed link a match writes. The vocabulary is the itd-84 +// discipline's (supersedes / reverses / duplicates / refines); a lexical +// score can only ever propose the two below. A reversal or a supersession is a +// judgement about meaning, which the discipline keeps advisory and human, so +// this package never proposes either. +type Relation string + +const ( + // Duplicates: the two texts hold each other's terms, both ways. + Duplicates Relation = "duplicates" + // Refines: the candidate holds the new text's terms, and carries a good + // deal more besides — the new record is the narrower of the two. + Refines Relation = "refines" +) + +// Bundled defaults. The threshold is configuration (the layered resolver's +// match.threshold); the rest are declared constants of the heuristic. +const ( + // DefaultThreshold is the weighted share of the new text's terms a + // candidate must hold for a link to be written. It was read off this + // repository's own ledger on 2026-09-26: each of the 449 open issues was + // matched against the other 1,660 issues and intents, and 14 cleared 0.6. + // Most of those were the same finding filed twice or a narrower follow-up + // of an earlier one; the false ones shared a relocation footer rather than + // a finding. Below 0.6 the best pairs stop being the same finding. + DefaultThreshold = 0.6 + // MinTerms is the fewest distinct terms a new text needs before it is + // compared at all. Below it the overlap of a line or two is chance, so the + // record is filed without matching and the outcome says so (the intent's + // scope condition). + MinTerms = 8 + // MaxLinks caps the links one filing writes. A text that clears the + // threshold against more candidates than this links the highest scorers; + // the rest are still listed, so nothing is hidden. + MaxLinks = 3 + // NearMissLimit caps how many below-threshold candidates an outcome lists. + NearMissLimit = 5 + // sharedLimit caps the shared terms shown per scored candidate. + sharedLimit = 8 + // minTermRunes is the shortest token that counts as a term. + minTermRunes = 3 +) + +// Candidate is one existing record's comparable text. +type Candidate struct { + ID string + Text string +} + +// Score is one candidate's result. +type Score struct { + ID string `json:"id"` + // Score is the weighted share of the NEW text's terms this candidate holds, + // in [0, 1] and rounded to three places. It is what the threshold compares. + Score float64 `json:"score"` + // Reverse is the weighted share of the CANDIDATE's terms the new text + // holds: high both ways is a duplicate, high one way a refinement. + Reverse float64 `json:"reverse"` + // Relation is set on a match above the threshold, and empty on a near miss. + Relation Relation `json:"relation,omitempty"` + // Linked is true when the writer wrote this match onto the record: every + // match up to MaxLinks. + Linked bool `json:"linked"` + // Shared are the rarest terms the two have in common, the evidence a person + // reads before confirming or removing the link. + Shared []string `json:"shared_terms"` +} + +// Outcome is one text matched against a candidate set. +type Outcome struct { + Heuristic string `json:"heuristic"` + Threshold float64 `json:"threshold"` + // Terms is the number of distinct terms the new text carries. + Terms int `json:"terms"` + MinTerms int `json:"min_terms"` + // Compared is the number of candidates scored. + Compared int `json:"compared"` + // Skipped says why no candidate was compared; empty when matching ran. + Skipped string `json:"skipped,omitempty"` + // Matches are the candidates at or above the threshold, best first. + Matches []Score `json:"matches"` + // NearMisses are the best candidates below it, with their scores. + NearMisses []Score `json:"near_misses"` +} + +// Links returns the typed links an outcome writes, relation → ids, best first. +func (o Outcome) Links() map[Relation][]string { + out := map[Relation][]string{} + for _, m := range o.Matches { + if m.Linked { + out[m.Relation] = append(out[m.Relation], m.ID) + } + } + return out +} + +// stopwords are the English function words that carry no finding. The list is +// short on purpose: domain words common to this record ("record", "intent") +// are discounted by their weight, not by a list. +var stopwords = func() map[string]bool { + m := map[string]bool{} + for _, w := range strings.Fields(`the and for are but not you all any can had her was one our out has + have been from they this that with which when what where who will would there their them then than + into onto its it's also only such each more most some very just over under about after before + because while does did doing done being were should could may might must shall these those here + upon via per nor yet both either neither own same other again once why how off too`) { + m[w] = true + } + return m +}() + +// Terms is the canonical tokeniser: lower-cased runs of letters and digits, +// at least three runes long, neither a stop word nor all digits (a count, a +// date or a record number says nothing about what a finding is), each once, +// sorted. +func Terms(text string) []string { + seen := map[string]bool{} + var out []string + flush := func(b *strings.Builder) { + t := b.String() + b.Reset() + if len([]rune(t)) < minTermRunes || stopwords[t] || allDigits(t) || seen[t] { + return + } + seen[t] = true + out = append(out, t) + } + var b strings.Builder + for _, r := range text { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(unicode.ToLower(r)) + continue + } + flush(&b) + } + flush(&b) + sort.Strings(out) + return out +} + +func allDigits(s string) bool { + for _, r := range s { + if !unicode.IsDigit(r) { + return false + } + } + return true +} + +// Weights is the rarity weight of every term across a candidate set: a term +// few candidates carry says more about a text than one most of them carry. A +// term no candidate carries takes the highest weight. +type Weights struct { + df map[string]int + n int +} + +// NewWeights counts, per term, how many of the term sets carry it. +func NewWeights(sets [][]string) Weights { + w := Weights{df: map[string]int{}, n: len(sets)} + for _, s := range sets { + for _, t := range s { + w.df[t]++ + } + } + return w +} + +// Of is a term's weight: a smoothed inverse document frequency, always > 0. +func (w Weights) Of(term string) float64 { + return math.Log(float64(w.n+1)/float64(w.df[term]+1)) + 1 +} + +// Overlap is THE overlap score: the weighted share of a's terms that b holds +// (forward), the weighted share of b's terms that a holds (reverse), and the +// shared terms, rarest first. Both inputs are sorted term sets (Terms). +func Overlap(a, b []string, w Weights) (forward, reverse float64, shared []string) { + var wa, wb, ws float64 + i, j := 0, 0 + for _, t := range a { + wa += w.Of(t) + } + for _, t := range b { + wb += w.Of(t) + } + for i < len(a) && j < len(b) { + switch { + case a[i] == b[j]: + ws += w.Of(a[i]) + shared = append(shared, a[i]) + i++ + j++ + case a[i] < b[j]: + i++ + default: + j++ + } + } + if wa > 0 { + forward = ws / wa + } + if wb > 0 { + reverse = ws / wb + } + sort.SliceStable(shared, func(x, y int) bool { + wx, wy := w.Of(shared[x]), w.Of(shared[y]) + if wx != wy { + return wx > wy + } + return shared[x] < shared[y] + }) + return forward, reverse, shared +} + +// Rank matches text against every candidate. A threshold outside (0, 1] is +// the caller's fault and reads as the bundled default, so a matcher can never +// be configured into linking everything or nothing by accident; the layered +// reader refuses such a value before it gets here. +func Rank(text string, cands []Candidate, threshold float64) Outcome { + if !(threshold > 0 && threshold <= 1) { + threshold = DefaultThreshold + } + terms := Terms(text) + o := Outcome{ + Heuristic: Heuristic, Threshold: threshold, Terms: len(terms), MinTerms: MinTerms, + Matches: []Score{}, NearMisses: []Score{}, + } + if len(terms) < MinTerms { + o.Skipped = "the text carries fewer distinct terms than the declared minimum, so it is filed without matching" + return o + } + if len(cands) == 0 { + o.Skipped = "there is no record to compare it with" + return o + } + sets := make([][]string, len(cands)) + for i, c := range cands { + sets[i] = Terms(c.Text) + } + w := NewWeights(sets) + var scored []Score + for i, c := range cands { + if len(sets[i]) == 0 { + continue + } + o.Compared++ + fwd, rev, shared := Overlap(terms, sets[i], w) + if len(shared) == 0 { + continue + } + if len(shared) > sharedLimit { + shared = shared[:sharedLimit] + } + // The rounded values are the ones compared, so a score shown as 0.6 is + // never a near miss of a 0.6 threshold. + s := Score{ID: c.ID, Score: round3(fwd), Reverse: round3(rev), Shared: shared} + if s.Score >= threshold { + s.Relation = Refines + if s.Reverse >= threshold { + s.Relation = Duplicates + } + } + scored = append(scored, s) + } + sort.SliceStable(scored, func(i, j int) bool { + if scored[i].Score != scored[j].Score { + return scored[i].Score > scored[j].Score + } + return scored[i].ID < scored[j].ID + }) + for _, s := range scored { + if s.Relation != "" { + s.Linked = len(o.Matches) < MaxLinks + o.Matches = append(o.Matches, s) + continue + } + if len(o.NearMisses) < NearMissLimit { + o.NearMisses = append(o.NearMisses, s) + } + } + return o +} + +func round3(f float64) float64 { return math.Round(f*1000) / 1000 } diff --git a/internal/core/record/match/match_test.go b/internal/core/record/match/match_test.go new file mode 100644 index 000000000..58afb90a9 --- /dev/null +++ b/internal/core/record/match/match_test.go @@ -0,0 +1,168 @@ +package match + +import ( + "reflect" + "strings" + "testing" +) + +// The fixtures read like ledger records: a finding, a near-identical refiling +// of it, a narrower follow-up of a broader one, and records about something +// else entirely. +const ( + finding = "The capture ledger reader silently skips a record whose frontmatter carries " + + "a duplicated key, so the finding disappears from every listing without a warning." + refiled = "Capture ledger reader silently skips any record whose frontmatter carries a " + + "duplicated key: the finding disappears from every listing, and no warning is printed." + broad = "Record readers disagree with the lint gate. The capture ledger reader skips " + + "records with duplicated frontmatter keys, the intent loader fails closed on a missing id, " + + "the spec store tolerates unknown properties, the site builder renders stale anchors, " + + "the memory store truncates pages, and the history store drops transcripts over its budget." + narrow = "The capture ledger reader skips records with duplicated frontmatter keys " + + "while the lint gate reports them." + unrelated1 = "The site builder renders a stale anchor for a heading renamed since the last build." + unrelated2 = "The history store drops a transcript that exceeds its byte budget without saying so." + unrelated3 = "The guard refuses a shell command whose here-document is never terminated." +) + +func corpus(extra ...Candidate) []Candidate { + return append([]Candidate{ + {ID: "iss-11", Text: unrelated1}, + {ID: "iss-12", Text: unrelated2}, + {ID: "itd-13", Text: unrelated3}, + }, extra...) +} + +func TestTermsIsTheCanonicalTokeniser(t *testing.T) { + got := Terms("The CAPTURE ledger, the capture Ledger: 2026 iss-42 a to é-dit x1y") + want := []string{"capture", "dit", "iss", "ledger", "x1y"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("Terms = %v, want %v (lower-cased, deduplicated, sorted; stop words, "+ + "short tokens and all-digit tokens dropped)", got, want) + } +} + +func TestOverlapIsDirectional(t *testing.T) { + a := Terms("alpha bravo charlie") + b := Terms("alpha bravo charlie delta echo foxtrot") + w := NewWeights([][]string{a, b}) + fwd, rev, shared := Overlap(a, b, w) + if fwd != 1 { + t.Fatalf("forward = %v, want 1: b holds every term of a", fwd) + } + if rev >= 1 || rev <= 0 { + t.Fatalf("reverse = %v, want strictly between 0 and 1", rev) + } + if len(shared) != 3 { + t.Fatalf("shared = %v, want the three common terms", shared) + } +} + +func TestANearIdenticalTextIsADuplicate(t *testing.T) { + o := Rank(refiled, corpus(Candidate{ID: "iss-7", Text: finding}), DefaultThreshold) + if o.Skipped != "" { + t.Fatalf("skipped: %s", o.Skipped) + } + if len(o.Matches) != 1 || o.Matches[0].ID != "iss-7" || o.Matches[0].Relation != Duplicates || !o.Matches[0].Linked { + t.Fatalf("matches = %+v, want iss-7 linked as duplicates", o.Matches) + } + if got := o.Links(); !reflect.DeepEqual(got, map[Relation][]string{Duplicates: {"iss-7"}}) { + t.Fatalf("links = %v", got) + } + if o.Heuristic == "" || !strings.Contains(o.Heuristic, "heuristic") { + t.Fatalf("the outcome does not declare itself a heuristic: %q", o.Heuristic) + } +} + +func TestANarrowerTextRefinesTheBroaderOne(t *testing.T) { + o := Rank(narrow, corpus(Candidate{ID: "itd-9", Text: broad}), DefaultThreshold) + if len(o.Matches) != 1 || o.Matches[0].ID != "itd-9" || o.Matches[0].Relation != Refines { + t.Fatalf("matches = %+v, want itd-9 as refines", o.Matches) + } + if o.Matches[0].Score < DefaultThreshold || o.Matches[0].Reverse >= DefaultThreshold { + t.Fatalf("score %v / reverse %v: a refinement is high one way and low the other", + o.Matches[0].Score, o.Matches[0].Reverse) + } +} + +func TestBelowTheThresholdNothingLinksAndTheNearMissesCarryScores(t *testing.T) { + o := Rank(finding, corpus(), DefaultThreshold) + if len(o.Matches) != 0 { + t.Fatalf("unrelated records matched: %+v", o.Matches) + } + if len(o.Links()) != 0 { + t.Fatalf("links written below the threshold: %v", o.Links()) + } + if len(o.NearMisses) == 0 { + t.Fatal("no near misses listed, though the unrelated records share terms with the text") + } + for _, nm := range o.NearMisses { + if nm.Score <= 0 || nm.Score >= DefaultThreshold || nm.Relation != "" || nm.Linked { + t.Fatalf("near miss %+v: want a score in (0, threshold), no relation, not linked", nm) + } + } + if o.Compared != 3 { + t.Fatalf("compared = %d, want 3", o.Compared) + } +} + +func TestTheThresholdIsHonoured(t *testing.T) { + // The same pair links at the default and does not at a threshold above + // its score: the threshold is the configuration, not a constant. + o := Rank(narrow, corpus(Candidate{ID: "itd-9", Text: broad}), DefaultThreshold) + s := o.Matches[0].Score + high := Rank(narrow, corpus(Candidate{ID: "itd-9", Text: broad}), s+0.01) + if len(high.Matches) != 0 || len(high.NearMisses) == 0 || high.NearMisses[0].ID != "itd-9" { + t.Fatalf("threshold %v: matches %+v near misses %+v", s+0.01, high.Matches, high.NearMisses) + } + if high.Threshold != s+0.01 { + t.Fatalf("outcome threshold = %v", high.Threshold) + } +} + +func TestAnOutOfRangeThresholdReadsAsTheDefault(t *testing.T) { + for _, th := range []float64{0, -1, 1.5} { + if o := Rank(finding, corpus(), th); o.Threshold != DefaultThreshold { + t.Fatalf("threshold %v became %v, want the default", th, o.Threshold) + } + } +} + +func TestAShortTextIsFiledWithoutMatchingAndSaysSo(t *testing.T) { + o := Rank("Typo in the ledger README.", corpus(Candidate{ID: "iss-7", Text: finding}), DefaultThreshold) + if o.Skipped == "" || !strings.Contains(o.Skipped, "minimum") { + t.Fatalf("skipped = %q, want the declared-minimum reason", o.Skipped) + } + if o.Compared != 0 || len(o.Matches) != 0 || len(o.NearMisses) != 0 { + t.Fatalf("a text below the minimum was compared: %+v", o) + } + if o.MinTerms != MinTerms || o.Terms >= MinTerms { + t.Fatalf("terms %d / min %d", o.Terms, o.MinTerms) + } +} + +func TestAnEmptyCorpusSaysSo(t *testing.T) { + if o := Rank(finding, nil, DefaultThreshold); o.Skipped == "" { + t.Fatal("an empty candidate set was not reported") + } +} + +func TestLinksAreCappedAndTheRestStillListed(t *testing.T) { + var extra []Candidate + for _, id := range []string{"iss-1", "iss-2", "iss-3", "iss-4", "iss-5"} { + extra = append(extra, Candidate{ID: id, Text: finding}) + } + o := Rank(refiled, corpus(extra...), DefaultThreshold) + if len(o.Matches) != 5 { + t.Fatalf("matches = %d, want all 5 listed", len(o.Matches)) + } + linked := 0 + for _, m := range o.Matches { + if m.Linked { + linked++ + } + } + if linked != MaxLinks || len(o.Links()[Duplicates]) != MaxLinks { + t.Fatalf("linked = %d, want MaxLinks (%d)", linked, MaxLinks) + } +} From b3690f07274d8563dae0606831c22872315fb98f Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:55:42 +0100 Subject: [PATCH 04/60] feat(record): match.threshold and match.fields through the layered reader LoadConfig resolves the two keys through internal/core/layered, the one layered configuration reader (.abcd/config.json, then ~/.abcd/config.json, then the bundled default), claiming the `match` namespace so a misspelt key is refused, and refusing a threshold outside (0, 1] or a field outside the closed set (issue.body, intent.title, intent.press_release) naming the file it came from. The reader already named match.threshold as a consumer it was built for; this is the first reader of the shared config family. Assisted-by: Claude:claude-opus-5-5 --- internal/core/record/match/config.go | 102 ++++++++++++++++++++++ internal/core/record/match/config_test.go | 91 +++++++++++++++++++ internal/core/record/match/match.go | 6 +- 3 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 internal/core/record/match/config.go create mode 100644 internal/core/record/match/config_test.go diff --git a/internal/core/record/match/config.go b/internal/core/record/match/config.go new file mode 100644 index 000000000..1dad86c89 --- /dev/null +++ b/internal/core/record/match/config.go @@ -0,0 +1,102 @@ +package match + +import ( + "fmt" + "strings" + + "github.com/intentdriven/abcd/internal/core/layered" +) + +// The compared fields: which text of each candidate family the match reads. +// The set is closed; a name outside it is refused, never ignored. +const ( + // FieldIssueBody is an open or resolved issue's markdown body. + FieldIssueBody = "issue.body" + // FieldIntentTitle is an intent's H1. + FieldIntentTitle = "intent.title" + // FieldIntentPressRelease is an intent's `## Press Release` section. + FieldIntentPressRelease = "intent.press_release" +) + +// DefaultFields is the bundled match.fields: every field in the set. +var DefaultFields = []string{FieldIssueBody, FieldIntentTitle, FieldIntentPressRelease} + +var knownFields = map[string]bool{FieldIssueBody: true, FieldIntentTitle: true, FieldIntentPressRelease: true} + +// Config is the resolved match configuration, with the origin of each value +// (the layered resolver's: "bundled", a file, or a flag), so a surface can say +// where a threshold came from. +type Config struct { + Threshold float64 `json:"threshold"` + ThresholdOrigin string `json:"threshold_origin"` + Fields []string `json:"fields"` + FieldsOrigin string `json:"fields_origin"` +} + +// Bundled is the configuration when nothing is configured. +func Bundled() Config { + return Config{ + Threshold: DefaultThreshold, ThresholdOrigin: "bundled", + Fields: append([]string(nil), DefaultFields...), FieldsOrigin: "bundled", + } +} + +// Compares reports whether the configuration reads field. +func (c Config) Compares(field string) bool { + for _, f := range c.Fields { + if f == field { + return true + } + } + return false +} + +// LoadConfig resolves match.threshold and match.fields through the one layered +// configuration reader (.abcd/config.json, then ~/.abcd/config.json, then the +// bundled defaults). It claims the `match` namespace, so a misspelt key is +// refused rather than letting a default apply unannounced, and a value outside +// its range or set is refused naming its file, never replaced by the default. +func LoadConfig(r layered.Roots) (Config, error) { + s, err := layered.Load(layered.Config, r) + if err != nil { + return Config{}, err + } + if err := s.Claim("match", "threshold", "fields"); err != nil { + return Config{}, err + } + th, err := layered.Get(s, "match.threshold", DefaultThreshold, func(v float64) error { + if !(v > 0 && v <= 1) { + return fmt.Errorf("want a share of the new text's terms greater than 0 and at most 1") + } + return nil + }) + if err != nil { + return Config{}, err + } + fields, err := layered.Get(s, "match.fields", append([]string(nil), DefaultFields...), checkFields) + if err != nil { + return Config{}, err + } + return Config{ + Threshold: th.V, ThresholdOrigin: th.Origin, + Fields: fields.V, FieldsOrigin: fields.Origin, + }, nil +} + +func checkFields(fs []string) error { + accepted := strings.Join(DefaultFields, ", ") + if len(fs) == 0 { + return fmt.Errorf("want at least one of %s", accepted) + } + seen := map[string]bool{} + for _, f := range fs { + if !knownFields[f] { + return fmt.Errorf("%q is not a compared field; want one or more of %s", layered.BoundKey(f), accepted) + } + if seen[f] { + return fmt.Errorf("%q is named twice", f) + } + seen[f] = true + } + return nil +} diff --git a/internal/core/record/match/config_test.go b/internal/core/record/match/config_test.go new file mode 100644 index 000000000..f690ac0b8 --- /dev/null +++ b/internal/core/record/match/config_test.go @@ -0,0 +1,91 @@ +package match + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/core/layered" +) + +// roots lays a checkout and a home under one temp dir, writing the repository +// and machine config files when given (0o600, which the machine layer's +// declaration guard admits). +func roots(t *testing.T, repoJSON, machineJSON string) layered.Roots { + t.Helper() + base := t.TempDir() + r := layered.Roots{Repo: filepath.Join(base, "repo"), Home: filepath.Join(base, "home")} + put := func(p, body string) { + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + if err := os.MkdirAll(r.Repo, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(r.Home, 0o700); err != nil { + t.Fatal(err) + } + if repoJSON != "" { + put(filepath.Join(r.Repo, ".abcd", "config.json"), repoJSON) + } + if machineJSON != "" { + put(filepath.Join(r.Home, ".abcd", "config.json"), machineJSON) + } + return r +} + +func TestConfigDefaultsWhenNothingIsConfigured(t *testing.T) { + c, err := LoadConfig(roots(t, `{"docs":{"target":"agents_md"}}`, "")) + if err != nil { + t.Fatal(err) + } + if c.Threshold != DefaultThreshold || !reflect.DeepEqual(c.Fields, DefaultFields) { + t.Fatalf("config = %+v, want the bundled defaults", c) + } + if c.ThresholdOrigin != "bundled" || c.FieldsOrigin != "bundled" { + t.Fatalf("origins = %q / %q, want bundled", c.ThresholdOrigin, c.FieldsOrigin) + } +} + +func TestConfigReadsThroughTheLayers(t *testing.T) { + c, err := LoadConfig(roots(t, + `{"match":{"threshold":0.75}}`, + `{"match":{"threshold":0.9,"fields":["issue.body"]}}`)) + if err != nil { + t.Fatal(err) + } + if c.Threshold != 0.75 || c.ThresholdOrigin != ".abcd/config.json" { + t.Fatalf("threshold %v from %q, want 0.75 from the repo file", c.Threshold, c.ThresholdOrigin) + } + if !reflect.DeepEqual(c.Fields, []string{"issue.body"}) || c.FieldsOrigin != "~/.abcd/config.json" { + t.Fatalf("fields %v from %q, want [issue.body] from the machine file", c.Fields, c.FieldsOrigin) + } + if !c.Compares(FieldIssueBody) || c.Compares(FieldIntentTitle) { + t.Fatalf("Compares disagrees with fields %v", c.Fields) + } +} + +func TestConfigRefusesWhatWouldSilentlyDoLess(t *testing.T) { + cases := map[string]string{ + `{"match":{"treshold":0.7}}`: "match.treshold", + `{"match":{"threshold":0}}`: "threshold", + `{"match":{"threshold":1.5}}`: "threshold", + `{"match":{"threshold":"high"}}`: "threshold", + `{"match":{"fields":[]}}`: "fields", + `{"match":{"fields":["issue.title"]}}`: "issue.title", + `{"match":{"fields":["issue.body","issue.body"]}}`: "twice", + `{"match":{"fields":["intent.title","intent.body"]}}`: "intent.body", + } + for repo, want := range cases { + _, err := LoadConfig(roots(t, repo, "")) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s: err = %v, want a refusal naming %q", repo, err, want) + } + } +} diff --git a/internal/core/record/match/match.go b/internal/core/record/match/match.go index 6b076d4ba..ab105e260 100644 --- a/internal/core/record/match/match.go +++ b/internal/core/record/match/match.go @@ -13,8 +13,10 @@ // capture and intent stores, and both of those call this package; a leaf // package is what lets every caller reach it without an import cycle. // -// The package reads nothing, writes nothing and never prints. Its callers -// gather the candidate texts under their own locks and write the links. +// The scorer reads nothing, writes nothing and never prints; the one read in +// the package is LoadConfig, through the layered configuration reader. Its +// callers gather the candidate texts under their own locks and write the +// links. package match import ( From edef56854952d30b947e4ab8d3a0387874d04481 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:00:55 +0100 Subject: [PATCH 05/60] feat: explain a missing gitleaks or gh in the refusal itself The two refusals abcd gives on main for a missing tool now carry the tool registry's explanation through tools.MissingError, which keeps the cause for errors.Is and appends what the tool is, whether the capability requires it, what works without it, the exact install step and what that install does. - history: a repository that armed gitleaks and has no binary still refuses to store a transcript (fail-closed), and the refusal names the way back to the native scanner (enabled: false). A binary that exists but is refused is not a missing tool and gets no install step. - ahoy remote and site setup: a missing gh is explained under the capability GitHubSettings, including that the install does not sign anyone in. This is itd-63's missing-scanner path: the pluggable safety gate the intent names as the first consumer (itd-62) is a draft, and its always-block-on-a-missing-scanner path does not exist on main; the gitleaks opt-in in the history store is the one fail-closed missing-scanner path there is. Assisted-by: Claude:claude-opus-5-5 --- internal/core/ahoy/remote.go | 7 ++- internal/core/ahoy/remote_gh_explain_test.go | 35 +++++++++++ .../core/history/gitleaks_explain_test.go | 59 +++++++++++++++++++ internal/core/history/history.go | 8 +++ internal/core/tools/explain.go | 20 +++++++ internal/core/tools/registry.go | 15 ++--- internal/core/tools/tools_test.go | 16 ++++- 7 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 internal/core/ahoy/remote_gh_explain_test.go create mode 100644 internal/core/history/gitleaks_explain_test.go diff --git a/internal/core/ahoy/remote.go b/internal/core/ahoy/remote.go index 8325880b6..c9bcb5656 100644 --- a/internal/core/ahoy/remote.go +++ b/internal/core/ahoy/remote.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "github.com/intentdriven/abcd/internal/core/tools" "github.com/intentdriven/abcd/internal/fsutil" ) @@ -455,8 +456,10 @@ func ghEnable(cwd, repo, key string) error { // the actor. func runGH(cwd string, stdin []byte, args ...string) ([]byte, error) { if _, err := exec.LookPath("gh"); err != nil { - return nil, errors.New("the GitHub CLI (gh) is not on PATH; abcd speaks to GitHub through it so that a " + - "remote write is made by your own authenticated identity, never by a token abcd holds") + // The refusal carries the tool registry's explanation (itd-63): what gh + // is, that these verbs require it, and the exact install step. + return nil, tools.Missing(errors.New("the GitHub CLI (gh) is not on PATH; abcd speaks to GitHub through it so that a "+ + "remote write is made by your own authenticated identity, never by a token abcd holds"), "gh", tools.GitHubSettings) } ctx, cancel := context.WithTimeout(context.Background(), remoteAPITimeout) defer cancel() diff --git a/internal/core/ahoy/remote_gh_explain_test.go b/internal/core/ahoy/remote_gh_explain_test.go new file mode 100644 index 000000000..0b6f6b347 --- /dev/null +++ b/internal/core/ahoy/remote_gh_explain_test.go @@ -0,0 +1,35 @@ +package ahoy + +import ( + "errors" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/core/tools" +) + +// emptyPath points PATH at an empty directory so no external tool is found. +func emptyPath(t *testing.T) { + t.Helper() + t.Setenv("PATH", t.TempDir()) +} + +// TestMissingGhIsExplained is itd-63 criterion 1 at the remote verbs: a missing +// gh is named with the registry's explanation, not a bare sentence. +func TestMissingGhIsExplained(t *testing.T) { + emptyPath(t) + _, err := runGH(t.TempDir(), nil, "api", "repos/example/example") + if err == nil { + t.Fatal("runGH succeeded with no gh on PATH") + } + var missing *tools.MissingError + if !errors.As(err, &missing) || missing.Explanation.Tool != "gh" { + t.Fatalf("error carries no registry explanation: %v", err) + } + e := tools.Explain("gh", tools.GitHubSettings) + for _, want := range []string{"not on PATH", "required for", e.StepText(), "gh auth login"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error lacks %q:\n%s", want, err) + } + } +} diff --git a/internal/core/history/gitleaks_explain_test.go b/internal/core/history/gitleaks_explain_test.go new file mode 100644 index 000000000..64bd59531 --- /dev/null +++ b/internal/core/history/gitleaks_explain_test.go @@ -0,0 +1,59 @@ +package history + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/adapter/gitleaks" + "github.com/intentdriven/abcd/internal/adapter/scanner" + "github.com/intentdriven/abcd/internal/core/tools" +) + +// TestCaptureExplainsTheMissingGitleaks is itd-63 criterion 4 at the one +// missing-scanner path on main: the transcript store of a repository that armed +// gitleaks. The refusal stands (fail-closed, nothing stored), and it now says +// what gitleaks is, that this repository requires it, the exact install step, +// and the way back to the native scanner, rather than a bare error. +func TestCaptureExplainsTheMissingGitleaks(t *testing.T) { + repoRoot, _ := setupStore(t) + restore := scanGitleaks + t.Cleanup(func() { scanGitleaks = restore }) + scanGitleaks = func(_, _, _ string) ([]scanner.Finding, error) { + return nil, fmt.Errorf("%w: not on PATH and no path configured", gitleaks.ErrConfiguredNotFound) + } + _, err := Capture(repoRoot, testRootSHA, []byte("user: hi\n"), CaptureMeta{SessionID: "sess-explain", Kind: "native"}) + if err == nil { + t.Fatal("capture did not fail closed") + } + if !errors.Is(err, gitleaks.ErrConfiguredNotFound) { + t.Fatalf("the explanation lost the sentinel: %v", err) + } + var missing *tools.MissingError + if !errors.As(err, &missing) || missing.Explanation.Capability != tools.TranscriptScanArmed { + t.Fatalf("error carries no registry explanation: %v", err) + } + e := tools.Explain("gitleaks", tools.TranscriptScanArmed) + for _, want := range []string{"gitleaks configured but not found", "required for", e.StepText(), "enabled to false"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal lacks %q:\n%s", want, err) + } + } +} + +// TestCaptureDoesNotExplainARefusedPath: a binary that exists but is refused +// is not a missing tool, and gets no install offer. +func TestCaptureDoesNotExplainARefusedPath(t *testing.T) { + repoRoot, _ := setupStore(t) + restore := scanGitleaks + t.Cleanup(func() { scanGitleaks = restore }) + scanGitleaks = func(_, _, _ string) ([]scanner.Finding, error) { + return nil, gitleaks.ErrConfiguredPathRefused + } + _, err := Capture(repoRoot, testRootSHA, []byte("user: hi\n"), CaptureMeta{SessionID: "sess-refused", Kind: "native"}) + var missing *tools.MissingError + if err == nil || errors.As(err, &missing) { + t.Fatalf("a refused path was explained as a missing tool: %v", err) + } +} diff --git a/internal/core/history/history.go b/internal/core/history/history.go index cd8fa2dca..5e3880259 100644 --- a/internal/core/history/history.go +++ b/internal/core/history/history.go @@ -33,6 +33,7 @@ import ( "time" "github.com/intentdriven/abcd/internal/adapter/gitleaks" + "github.com/intentdriven/abcd/internal/core/tools" "github.com/intentdriven/abcd/internal/adapter/scanner" "github.com/intentdriven/abcd/internal/fsutil" ) @@ -251,6 +252,13 @@ func Capture(repoRoot, rootSHA string, raw []byte, meta CaptureMeta) (CaptureRes // asked for. extra, err := scanGitleaks(repoRoot, text, "transcript") if err != nil { + // A binary the repository asked for and nobody installed is a missing + // tool: the refusal stands, and it says what gitleaks is, that this + // repository requires it, the install step, and the way back to the + // native scanner (itd-63). A refused path is not a missing tool. + if errors.Is(err, gitleaks.ErrConfiguredNotFound) { + err = tools.Missing(err, "gitleaks", tools.TranscriptScanArmed) + } return CaptureResult{}, fmt.Errorf("history: %w", err) } findings = append(findings, extra...) diff --git a/internal/core/tools/explain.go b/internal/core/tools/explain.go index 8945b0527..5045b7104 100644 --- a/internal/core/tools/explain.go +++ b/internal/core/tools/explain.go @@ -146,3 +146,23 @@ func (e Explanation) Lines() []string { } return lines } + +// MissingError is a verb's refusal for a missing tool with the registry's +// explanation appended: the refusal stands exactly as it was (its cause stays +// reachable through errors.Is), and only its message grows to say what the +// tool is, whether this capability needs it, and the exact install step. +type MissingError struct { + Cause error + Explanation Explanation +} + +// Missing wraps cause with the explanation for name as capability uses it. +func Missing(cause error, name string, capability Capability) error { + return &MissingError{Cause: cause, Explanation: Explain(name, capability)} +} + +func (m *MissingError) Error() string { + return m.Cause.Error() + "\n" + strings.Join(m.Explanation.Lines(), "\n") +} + +func (m *MissingError) Unwrap() error { return m.Cause } diff --git a/internal/core/tools/registry.go b/internal/core/tools/registry.go index e48c83caf..9f5975b38 100644 --- a/internal/core/tools/registry.go +++ b/internal/core/tools/registry.go @@ -44,9 +44,10 @@ const ( // in .abcd/config/gitleaks.json: the history store refuses to store a // transcript with less coverage than the repository asked for. TranscriptScanArmed Capability = "transcript-scan-armed" - // RemoteSettings is `ahoy remote`: reading and changing the repository's - // GitHub secret-scanning settings, which abcd does only through gh. - RemoteSettings Capability = "remote-settings" + // GitHubSettings is every verb that reads or changes the repository's + // settings on GitHub (ahoy remote, site setup), which abcd does only + // through gh. + GitHubSettings Capability = "github-settings" ) // Use is what one capability does with a tool, in the words a person reads. @@ -141,13 +142,13 @@ var registry = map[string]Tool{ What: "GitHub's own command-line program, signed in as you", Homepage: "https://cli.github.com", Uses: map[Capability]Use{ - RemoteSettings: { - Capability: "reading and changing this repository's GitHub secret-scanning settings (ahoy remote)", + GitHubSettings: { + Capability: "reading and changing this repository's settings on GitHub (ahoy remote, site setup)", Requirement: Required, Does: "abcd speaks to GitHub through gh, so a remote change is made by your own signed-in identity " + "and abcd never holds a token", - WithoutIt: "ahoy remote cannot read or change the settings; nothing else in abcd needs gh", - OnDecline: "the remote settings stay unread and unchanged", + WithoutIt: "ahoy remote and site setup cannot read or change the settings; nothing else in abcd needs gh", + OnDecline: "the settings on GitHub stay unread and unchanged", }, }, Install: map[string]Step{"darwin": homebrew("gh"), "linux": homebrew("gh")}, diff --git a/internal/core/tools/tools_test.go b/internal/core/tools/tools_test.go index 53c0dc4a8..ddf6d4ec8 100644 --- a/internal/core/tools/tools_test.go +++ b/internal/core/tools/tools_test.go @@ -52,7 +52,7 @@ func TestExplainIsPerCapability(t *testing.T) { if !strings.Contains(e.WithoutIt, "enabled") { t.Errorf("an armed repository's without-it must name the opt-out that returns it to the native scanner: %q", e.WithoutIt) } - gh := explainFor("gh", RemoteSettings, "darwin") + gh := explainFor("gh", GitHubSettings, "darwin") if gh.Requirement != Required || !gh.Known { t.Fatalf("gh for the remote settings: %+v", gh) } @@ -375,3 +375,17 @@ func TestRunArgvKillsItsOwnGroupOnTimeout(t *testing.T) { t.Fatalf("runArgv waited %s: the group was not killed", d) } } + +// TestMissingErrorKeepsTheCauseAndCarriesTheExplanation: a verb's refusal for a +// missing tool keeps its sentinel for errors.Is and appends the explanation. +func TestMissingErrorKeepsTheCauseAndCarriesTheExplanation(t *testing.T) { + cause := errors.New("gitleaks configured but not found") + err := Missing(cause, "gitleaks", TranscriptScanArmed) + if !errors.Is(err, cause) { + t.Fatal("the cause is lost") + } + lines := strings.Split(err.Error(), "\n") + if lines[0] != cause.Error() || len(lines) < 5 { + t.Fatalf("error = %q, want the cause then the explanation", err.Error()) + } +} From 920afe82fd48646a5f3697d2dd23b4eef4df6955 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:01:06 +0100 Subject: [PATCH 06/60] feat(ahoy): explain each missing tool and install it only on a yes ahoy's dependency gap and its install step now run through the explain-then-install mode (itd-63) instead of printing a bare "brew install gitleaks". - detect: the gitleaks gap carries the registry explanation (Gap.Tool, in --json) and is required where the repository armed gitleaks in .abcd/config/gitleaks.json, optional over the native scanner otherwise. - install: the category approval reaches the step; each missing tool is then put to the front door's confirmation. A yes runs the registry's step and reports it as a change with its verify result; a no, a failure or an unasked caller is a note carrying the explanation and "continuing on the native secret scanner". - CLI: the confirmation asks only at a terminal (default no), or takes --install-tool , the answer a host's question tool relays; a piped answer, --yes and CI never install. --install-tool refuses a name ahoy install does not check for, naming the ones it does. - The trufflehog gap is gone: nothing in abcd runs trufflehog, so the mode would have installed a program that does nothing for the person. The command page, the ahoy brief chapter, the sentence, the generated surface, the package map and ACKNOWLEDGEMENTS (Homebrew) move with it. Refs: iss-2609261447331434 Assisted-by: Claude:claude-opus-5-5 --- .../development/brief/04-surfaces/01-ahoy.md | 22 +- .abcd/development/release/surface.json | 9 +- ACKNOWLEDGEMENTS.md | 5 + commands/ahoy.md | 31 ++- docs/reference/cli/commands.md | 3 +- internal/README.md | 7 + internal/core/ahoy/ahoy.go | 17 +- internal/core/ahoy/apply.go | 35 ++- internal/core/ahoy/detect.go | 54 +++-- internal/core/ahoy/tools_route_test.go | 206 ++++++++++++++++++ internal/core/surface/sentences.go | 2 +- .../surface/cli/ahoy_tool_confirm_test.go | 90 ++++++++ internal/surface/cli/cli.go | 62 +++++- 13 files changed, 510 insertions(+), 33 deletions(-) create mode 100644 internal/core/ahoy/tools_route_test.go create mode 100644 internal/surface/cli/ahoy_tool_confirm_test.go diff --git a/.abcd/development/brief/04-surfaces/01-ahoy.md b/.abcd/development/brief/04-surfaces/01-ahoy.md index 2fe8a13fd..69f5ecc9e 100644 --- a/.abcd/development/brief/04-surfaces/01-ahoy.md +++ b/.abcd/development/brief/04-surfaces/01-ahoy.md @@ -333,7 +333,7 @@ about, one question per category present, never one per item. | `safe-autocreate` | the repo skeleton, history-store directories, the name-guard artefacts | applied once the category is approved, no per-item prompt; create-if-absent, never overwriting | | `config-change` | visibility, oracle adapter, the `PATH` entry, the git-identity pin | transparent confirm; skip-if-set with a "current value" notice | | `plugin-owned` | the marker block (itd-3); hook-manifest verification | silent overwrite on marker drift; a non-resolvable diagnostic for a malformed or missing manifest, and for a conventions file whose block would land inside a fence or HTML comment nothing closes (`marker.unplaceable`) | -| `dependency` | the opt-in scanners | one category-level approval covering them; abcd never auto-executes a package manager, and the user runs the commands | +| `dependency` | a tool a capability uses and cannot find: gitleaks, optional over the native secret scanner and required where the repository armed it in `.abcd/config/gitleaks.json` | the category approval reaches the step; each tool is then explained from the tool registry (what it is, optional or required here, what works without it, the exact install step, what the install does) and its install step runs only on a per-tool yes — typed at a terminal, or relayed by a host as a flag naming the tool — never under the approve-everything flag, a piped answer or CI; a no is reported as what the capability continues on | | `status-line` | the offer of abcd's status line in the host harness | an advisory offer asked after its own question, written only on an answered consent; never under the approve-everything flag, and reported as optional work it skipped | | `oracle-routing` | the offer of abcd's proposed model-tier routing table (itd-2609170822093401): the machine's `~/.abcd/oracle-routing.json`, then, as a separate question, the repository's `.abcd/config/oracle-routing.json` | the proposal rendered as a table (agent, tier, fan-out) and each file written only on its own answered consent, the machine one owner-only; never under the approve-everything flag, and reported as optional work it skipped; a decline records nothing, so the next install offers again; uninstall leaves both files | | `user-state` | the registry entry, re-founding, stale or duplicate entries | guided; never auto-edit user-scope state, report extras read-only | @@ -359,6 +359,16 @@ prompt wait rather than decline, which is the contract every prompting CLI has. A run that must neither block nor prompt closes stdin and pre-answers with flags. +**Installing a tool is the one question a piped answer never answers.** It runs +a program on the machine, so it is asked only of a person at a terminal, after +the tool registry's explanation is shown, and its default is no. Off a terminal +the answer is a flag naming the tool, which is how a host relays the answer its +own question tool got; the approve-everything flag never installs a tool, and a +run with `CI` set never installs one and is not asked. What runs is the +registry's fixed argv for the platform, never a shell string and never a command +composed from input, and only when the package manager resolves on `PATH` +outside the repository (`internal/core/tools`). + The non-interactive flags pre-answer the prompts: approve every resolvable category, decide the adoption question either way, set the marker target, the oracle backend, the deep-scan toggle and the repo visibility, select track-latest @@ -500,9 +510,12 @@ byte-identical to a fresh install save for the setup date. - **Given** a repo with the install run at an older setup version, **when** the install runs, **then** the version is updated, the marker block refreshed, and existing config keys preserved. -- **Given** an opt-in scanner is not on `PATH`, **when** the dependency category - is approved, **then** the user is shown the install commands under one - category-level approval; abcd never auto-executes a package manager. +- **Given** a tool a capability uses is not on `PATH`, **when** the dependency + category is approved, **then** the person is shown the tool registry's + explanation and asked per tool; the registry's fixed install step runs only + on a yes typed at a terminal or relayed by a host naming the tool, the result + reports what ran and whether its verify passed, and a no reports what the + capability continues on (itd-63). - **Given** no oracle adapter is wired, **when** detection resolves the oracle, **then** it stays host-delegated: abcd needs no API keys or model config, because it emits prompts the host runs (adr-25), and an adapter can be @@ -562,6 +575,7 @@ Sub-verbs: none. | `--bin-dir` | string | | `--dev` | bool | | `--docs-target` | string | +| `--install-tool` | stringSlice | | `--oracle-backend` | string | | `--refuse-adopt` | bool | | `--scan-deep` | string | diff --git a/.abcd/development/release/surface.json b/.abcd/development/release/surface.json index 1494215a2..e1dd9ca8b 100644 --- a/.abcd/development/release/surface.json +++ b/.abcd/development/release/surface.json @@ -87,7 +87,7 @@ { "path": "abcd ahoy install", "hidden": false, - "sentence": "Apply the install gaps the detection finds: Writes the .abcd/ scaffolding, the name-guard hooks, and the PATH entry; refuses a stale binary before any write.", + "sentence": "Apply the install gaps the detection finds: Writes .abcd/, the name-guard hooks and the PATH entry, and installs a tool only on a yes; refuses a stale binary.", "flags": [ { "name": "adopt", @@ -131,6 +131,13 @@ "required": false, "hidden": false }, + { + "name": "install-tool", + "shorthand": "", + "type": "stringSlice", + "required": false, + "hidden": false + }, { "name": "oracle-backend", "shorthand": "", diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index d4a621c98..244632042 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -166,6 +166,11 @@ Ideas and methodologies that shaped the design — not code abcd depends on. fix) while rejecting their implicit background network check: abcd implements the same grammar over disk-only sources, and the network answers only an explicit `--check` (adr-38). +- **Homebrew (BSD-2-Clause)** — the package manager the tool registry's install + steps run (`brew install `, on macOS and on Linux), chosen as the one + manager with a single fixed command on both supported platforms, so an + explained install is an argv abcd can show exactly and run only on a yes + (itd-63). - **git's editor hand-off (`GIT_EDITOR`, then `$VISUAL`, then `$EDITOR`)** — the order and the shape bare `abcd report` follows to open the report skeleton: `$VISUAL` before `$EDITOR`, run through the shell so the setting may carry diff --git a/commands/ahoy.md b/commands/ahoy.md index 0c498afee..9688fc6a4 100644 --- a/commands/ahoy.md +++ b/commands/ahoy.md @@ -61,7 +61,11 @@ Then summarise the JSON for the user: verbatim: it is the one sentence stating what the private layer does NOT cover, and a paraphrase drops the half that matters. - `gaps` — how many are outstanding, and for each actionable one its `title`, - `category`, and `fix_hint`; call out which are `required`. + `category`, and `fix_hint`; call out which are `required`. A `dependency` gap + carries `tool`, the tool registry's explanation: relay `what`, the + `requirement` for `capability_name`, `without_it`, `does`, the exact `step` + and `effects` (what the install does to the machine) rather than a bare + command, so the person can judge the install. If there are actionable gaps, tell the user to run `/abcd:ahoy install` to apply them. If `folder_kind` is `unmanaged-folder`, note there is nothing to act on @@ -144,6 +148,27 @@ harness-wide setting, and never accepts a model-tier routing table (below), because a table decides which model every delegated step asks for. When the result carries `optional_skipped`, report it and offer the `yes |` form above as the way to apply it. +**The tool question.** When a `dependency` gap is present and its category is +approved, each missing tool is its own question, and a piped answer never +answers it: installing runs a program on the machine. At a terminal the install +shows the explanation and asks `Install now by running ? [y/N]`. +Through this page, ask the user with the host's question tool instead: present +the gap's `tool` explanation (what it is, whether this capability needs it, what +works without it, the exact step, what the install does), and only on their yes +run + +```bash +"${CLAUDE_PLUGIN_ROOT}/abcd" ahoy install --install-tool --json +``` + +`--install-tool` is the relayed yes, for the named tool only; never pass it +without the user's answer. The result's `changes` reports what ran and whether +its verify passed; a no, a failed step or a missing package manager is a `notes` +line ending in what the capability continues on (`continuing on the native +secret scanner`). `--yes` never installs a tool, and a run with `CI` set +installs none and is not asked. A name that is not a tool `ahoy install` checks +for is refused, naming the ones it does. + **The house-style question.** When the install seeds `.abcd/docs-lint.json`, it asks `docs_lint.em_dash_in_list_item (blocking/warning) [warning]`: whether an em dash inside a list item, abcd's own house style rather than a currency rule, @@ -296,7 +321,9 @@ named. The call goes through the GitHub CLI (`gh`), so the write is made by the user's own authenticated identity and abcd never holds a token; if `gh` is absent the -verb refuses and says so. It is idempotent — a repository already in the desired +verb refuses, and the refusal carries the tool registry's explanation of `gh`: +what it is, that these verbs require it, the exact install step and what that +install does. Relay it; the step is the user's to run. It is idempotent — a repository already in the desired state takes no write, and a re-run rewrites nothing in the tree — and it stops at the first failed step rather than attempting one that cannot succeed. Relay `status`, the resolved `repo`, every `change`, and every `note`: a note is a diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index adc7dbb7e..0d83d9c5f 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -56,7 +56,7 @@ Report every install gap, user-scope state included: Writes nothing; refuses any #### `abcd ahoy install` -Apply the install gaps the detection finds: Writes the .abcd/ scaffolding, the name-guard hooks, and the PATH entry; refuses a stale binary before any write. +Apply the install gaps the detection finds: Writes .abcd/, the name-guard hooks and the PATH entry, and installs a tool only on a yes; refuses a stale binary. **Usage:** `abcd ahoy install [flags]` @@ -69,6 +69,7 @@ Apply the install gaps the detection finds: Writes the .abcd/ scaffolding, the n --bin-dir string directory for the PATH entry (default ~/.local/bin, or an existing abcd install adopted in place); fails when it is not writable — abcd never escalates privileges --dev track-latest dogfood mode: the PATH entry rebuilds from the source tip on every call instead of pinning the built binary --docs-target string which conventions file carries the managed block, which names abcd: claude_md | agents_md | both | skip (default skip) + --install-tool strings answer yes to installing this missing tool (repeatable): the answer a host's question tool relays; without it a tool is installed only on an answer typed at a terminal, never on the approve-everything flag, a piped answer or CI --oracle-backend string oracle backend: host-delegated | native | cli | api | mcp --refuse-adopt decline to adopt an unmanaged repo --scan-deep string enable deep scan: true | false diff --git a/internal/README.md b/internal/README.md index 74197df6a..76e0c7f37 100644 --- a/internal/README.md +++ b/internal/README.md @@ -117,6 +117,13 @@ plugin surface, and a future MCP server share one engine. and it holds the seam a specialist link checker would later slot into — the baseline schema and the lint rules are the contract, the fetcher is a replaceable producer. +- **`core/tools/`** — the explain-then-install mode (itd-63): the curated + registry of the external tools abcd knows, the plain-language explanation a + verb gives when one is missing, and the one place abcd runs a package + manager. It is a trust boundary, and a mode rather than a surface: a verb + hands `Install` a confirmation its front door supplies, and what runs is the + registry's fixed argv, never composed from input, never a shell, never in CI, + and never a program that resolves inside the repository. - **`core/lifeboat/`** — the brief↔lifeboat contract. `mapping.go` is the single source of truth for which brief section a lifeboat fills from which source tier, and it is rendered into the brief's `00-meta.md` with a test asserting diff --git a/internal/core/ahoy/ahoy.go b/internal/core/ahoy/ahoy.go index 9c6a98804..8c2de3252 100644 --- a/internal/core/ahoy/ahoy.go +++ b/internal/core/ahoy/ahoy.go @@ -12,6 +12,8 @@ // routed through the injected Prompter seam. package ahoy +import "github.com/intentdriven/abcd/internal/core/tools" + // FolderKind is the classification of the folder ahoy runs in. There is no // workspace layer: abcd manages exactly one kind of folder, a repository. type FolderKind string @@ -36,7 +38,10 @@ const ( ConfigChange GapCategory = "config-change" // PluginOwned covers the marker block and the (verify-only) hook manifest. PluginOwned GapCategory = "plugin-owned" - // Dependency covers opt-in scanners on PATH (surfaced, never auto-run). + // Dependency covers the external tools a capability uses (itd-63): each gap + // carries the tool registry's explanation, and a tool is installed only on + // an answer the front door's confirmation returns, never on the category + // approval alone. Dependency GapCategory = "dependency" // UserState covers ~/.abcd/history registry state (guided, never auto-edited). UserState GapCategory = "user-state" @@ -63,6 +68,10 @@ type Gap struct { FixHint string `json:"fix_hint"` Required bool `json:"required"` // advisory gaps set false Resolvable bool `json:"resolvable"` // false => diagnostic only + // Tool is the tool registry's explanation for a dependency gap (itd-63): + // what the tool is, whether this capability needs it, what works without + // it, and the exact install step. Absent on every other gap. + Tool *tools.Explanation `json:"tool,omitempty"` } // RepoIdentity is the deterministic identity of the repo under cwd. @@ -146,6 +155,12 @@ type InstallOptions struct { // cannot be determined, otherwise refuses before any write. The override is // the documented escape when a rebuild is not an option. --allow-stale-binary. AllowStaleBinary bool + // ConfirmTool is asked, once per missing tool, whether to run the tool + // registry's install step (itd-63). The front door supplies it: the CLI + // asks at a terminal, or answers yes for a tool the person named with + // --install-tool. Nil is a no for every tool, so a caller that asks nothing + // installs nothing. The category approval never stands in for it. + ConfirmTool tools.Confirm } // InstallResult is the outcome of Install. diff --git a/internal/core/ahoy/apply.go b/internal/core/ahoy/apply.go index da0a8d46d..1b8798bdd 100644 --- a/internal/core/ahoy/apply.go +++ b/internal/core/ahoy/apply.go @@ -14,6 +14,7 @@ import ( "github.com/intentdriven/abcd/internal/core/history" "github.com/intentdriven/abcd/internal/core/identity" + "github.com/intentdriven/abcd/internal/core/tools" ) // Install runs detect + apply over the approved categories. It is idempotent: @@ -132,6 +133,7 @@ func Install(cwd string, opts InstallOptions, p Prompter) (InstallResult, error) modeForced: modeForced, binTarget: binTargetPath, attribution: opts.Attribution, + confirmTool: opts.ConfirmTool, } // itd-111 refusal: a binary that is stale against its own source tip, or @@ -314,6 +316,7 @@ type applyCtx struct { modeForced bool // the requested install mode differs from the on-disk state attribution bool // --attribution: opt this repo into the committed prompt hook binTarget string // the resolved PATH entry this run installs (never re-derived) + confirmTool tools.Confirm visibilityForced bool // an explicit --visibility override overwrote a valid value docsTargetForced bool // a --docs-target override overwrote a valid value, or this run chose the first one @@ -407,23 +410,43 @@ func (a *applyCtx) stepIdentityPin() { func (a *applyCtx) has(id string) bool { return a.gapPresent[id] } -// stepDependencies re-probes PATH; surfaces the fix hint but never auto-runs a -// package manager. +// stepDependencies is the explain-then-install mode at ahoy (itd-63). For each +// dependency gap whose tool is still missing it hands the tool registry's +// explanation to the front door's confirmation, and runs the registry's step +// only on a yes. What ran, and whether it verified, is a change; a no, a failure +// or a caller that asked nothing is a note carrying the explanation and the +// capability's standing ("continuing on the native secret scanner"), never a +// silent skip. The category approval gates reaching this step at all; it never +// stands in for the per-tool answer. func (a *applyCtx) stepDependencies() { if !a.approved[Dependency] { return } for _, g := range a.det.Gaps { - if g.Category != Dependency { + if g.Category != Dependency || g.Tool == nil { continue } - tool := strings.TrimPrefix(strings.TrimSuffix(g.ID, "_missing"), "deps.") - if !onPath(tool) { - a.note("dependency: " + g.FixHint) + if onPath(g.Tool.Tool) { + continue + } + res := newToolInstaller(a.cwd).Install(g.Tool.Tool, g.Tool.Capability, a.confirmTool) + if res.Ran && res.Installed && res.Verified { + a.changes = append(a.changes, receiptPath(a.cwd, "dependency: "+res.Summary())) + continue } + if !res.Ran { + for _, line := range g.Tool.Lines() { + a.refuse(receiptPath(a.cwd, "dependency: "+line)) + } + } + a.refuse(receiptPath(a.cwd, "dependency: "+res.Summary())) } } +// newToolInstaller is the tool installer seam, guarded on the repository the +// install runs in; a test swaps it for one that records instead of executing. +var newToolInstaller = tools.Default + // stepSkeleton writes .abcd/config.json seed when the skeleton gap is present. func (a *applyCtx) stepSkeleton() { if !a.approved[SafeAutocreate] || !a.has("skeleton.config_missing") { diff --git a/internal/core/ahoy/detect.go b/internal/core/ahoy/detect.go index 49ffc2630..9bb0749a9 100644 --- a/internal/core/ahoy/detect.go +++ b/internal/core/ahoy/detect.go @@ -9,7 +9,9 @@ import ( "sort" "strings" + "github.com/intentdriven/abcd/internal/adapter/gitleaks" "github.com/intentdriven/abcd/internal/core/identity" + "github.com/intentdriven/abcd/internal/core/tools" ) // Enumerations for config-value validation. @@ -87,7 +89,7 @@ func Detect(cwd string) (DetectionResult, error) { var gaps []Gap gaps = append(gaps, detectPluginRoot(pluginOK)...) if kind != UnmanagedFolder { - gaps = append(gaps, detectDependencies()...) + gaps = append(gaps, detectDependencies(abs)...) gaps = append(gaps, detectSkeleton(abs)...) gaps = append(gaps, detectLocalTier(abs)...) gaps = append(gaps, detectIdentity(identity, idx)...) @@ -187,23 +189,43 @@ func detectPluginRoot(ok bool) []Gap { }} } -func detectDependencies() []Gap { - var gaps []Gap - if !onPath("gitleaks") { - gaps = append(gaps, Gap{ - ID: "deps.gitleaks_missing", Category: Dependency, Scope: "machine", - Title: "gitleaks not on PATH", Detail: "gitleaks enables a deeper secret scan.", - FixHint: "brew install gitleaks", Required: false, Resolvable: true, - }) +// DependencyTools names the tools detectDependencies checks for, and so the +// tools `ahoy install --install-tool` can name: the front door refuses any +// other, since no gap would ever put it to the question. +var DependencyTools = []string{"gitleaks"} + +// detectDependencies names each external tool a capability here would use and +// cannot find, with the tool registry's explanation rather than a bare command +// (itd-63). gitleaks is the one ahoy checks: optional over the native secret +// scanner, and REQUIRED in a repository that armed it in +// .abcd/config/gitleaks.json, whose transcript capture refuses without it. An +// armed repository that names an existing binary by path has no gap: the +// adapter judges that path itself, and a refusal there is not a missing tool. +// +// trufflehog is not offered: nothing in abcd runs it (iss-2609261447331434), +// and a gap asking a person to install a program abcd never uses is not an +// explanation anyone can act on. +func detectDependencies(cwd string) []Gap { + capability := tools.TranscriptScan + if cfg, err := gitleaks.LoadConfig(cwd); err == nil && cfg.Enabled { + capability = tools.TranscriptScanArmed + if p := strings.TrimSpace(cfg.Path); p != "" && fileExists(p) { + return nil + } } - if !onPath("trufflehog") { - gaps = append(gaps, Gap{ - ID: "deps.trufflehog_missing", Category: Dependency, Scope: "machine", - Title: "trufflehog not on PATH", Detail: "trufflehog enables deep secret scanning when scan.deep=true.", - FixHint: "brew install trufflehog", Required: false, Resolvable: true, - }) + if onPath("gitleaks") { + return nil } - return gaps + e := tools.Explain("gitleaks", capability) + return []Gap{{ + ID: "deps.gitleaks_missing", Category: Dependency, Scope: "machine", + Title: "gitleaks not on PATH", + Detail: string(e.Requirement) + " for " + e.CapabilityName + "; without it: " + e.WithoutIt + ".", + FixHint: e.StepText() + " (abcd ahoy install explains it and runs it only on your yes)", + Required: e.Requirement == tools.Required, + Resolvable: true, + Tool: &e, + }} } func onPath(tool string) bool { diff --git a/internal/core/ahoy/tools_route_test.go b/internal/core/ahoy/tools_route_test.go new file mode 100644 index 000000000..1dacde4ae --- /dev/null +++ b/internal/core/ahoy/tools_route_test.go @@ -0,0 +1,206 @@ +package ahoy + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/core/tools" +) + +func depGap(t *testing.T, gaps []Gap, id string) Gap { + t.Helper() + for _, g := range gaps { + if g.ID == id { + return g + } + } + t.Fatalf("no %s gap among %v", id, gaps) + return Gap{} +} + +// TestDependencyGapCarriesTheRegistryExplanation is itd-63 criterion 1 at +// ahoy's gap: the gitleaks gap states what the registry says, not a bare +// command. +func TestDependencyGapCarriesTheRegistryExplanation(t *testing.T) { + emptyPath(t) + g := depGap(t, detectDependencies(t.TempDir()), "deps.gitleaks_missing") + if g.Tool == nil || g.Tool.Tool != "gitleaks" || !g.Tool.Known { + t.Fatalf("gap carries no registry explanation: %+v", g) + } + if g.Required || g.Tool.Requirement != tools.Optional { + t.Fatalf("an unarmed repository's gitleaks gap reads as required: %+v", g) + } + if !strings.Contains(g.Detail, "native secret scanner") { + t.Errorf("detail does not say what works without it: %q", g.Detail) + } + if !strings.Contains(g.FixHint, g.Tool.StepText()) { + t.Errorf("fix hint %q does not carry the registry step %q", g.FixHint, g.Tool.StepText()) + } +} + +// TestArmedRepositoryMakesGitleaksRequired: a repository that armed gitleaks +// needs it, and the gap says so with the way back to the native scanner. +func TestArmedRepositoryMakesGitleaksRequired(t *testing.T) { + emptyPath(t) + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".abcd", "config"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, ".abcd", "config", "gitleaks.json"), + []byte(`{"schema_version":1,"enabled":true}`), 0o644); err != nil { + t.Fatal(err) + } + g := depGap(t, detectDependencies(repo), "deps.gitleaks_missing") + if !g.Required || g.Tool == nil || g.Tool.Capability != tools.TranscriptScanArmed { + t.Fatalf("armed repository's gap: %+v", g) + } + if !strings.Contains(g.Detail, "enabled to false") { + t.Errorf("detail does not name the way back: %q", g.Detail) + } +} + +// TestNoTrufflehogGap: nothing in abcd runs trufflehog, so ahoy never asks a +// person to install it (iss-2609261447331434). +func TestNoTrufflehogGap(t *testing.T) { + emptyPath(t) + for _, g := range detectDependencies(t.TempDir()) { + if strings.Contains(g.ID, "trufflehog") { + t.Fatalf("ahoy still offers trufflehog: %+v", g) + } + } +} + +// fakeToolRun is a tools.Installer whose programs all "exist" outside the +// repository and whose runs are recorded rather than executed. +func fakeToolRun(t *testing.T, calls *[][]string) { + t.Helper() + bin := t.TempDir() + for _, n := range []string{"brew", "gitleaks"} { + if err := os.WriteFile(filepath.Join(bin, n), []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + } + prev := newToolInstaller + newToolInstaller = func(guard string) *tools.Installer { + in := tools.Default(guard) + in.LookPath = func(n string) (string, error) { + p := filepath.Join(bin, n) + if _, err := os.Stat(p); err != nil { + return "", errors.New("absent") + } + return p, nil + } + in.Getenv = func(string) string { return "" } + in.Run = func(_ context.Context, argv []string) ([]byte, error) { + *calls = append(*calls, argv) + return []byte("8.0.0\n"), nil + } + return in + } + t.Cleanup(func() { newToolInstaller = prev }) +} + +func depCtx(t *testing.T, confirm tools.Confirm) *applyCtx { + t.Helper() + emptyPath(t) + repo := t.TempDir() + return &applyCtx{ + cwd: repo, + det: DetectionResult{Gaps: detectDependencies(repo)}, + approved: map[GapCategory]bool{Dependency: true}, + confirmTool: confirm, + } +} + +// TestDependencyNoKeepsTheNativeDefaultAndSaysSo is criterion 2's no half at +// ahoy: nothing runs, the explanation is in the notes, and the loud line names +// what the capability continues on. +func TestDependencyNoKeepsTheNativeDefaultAndSaysSo(t *testing.T) { + var calls [][]string + fakeToolRun(t, &calls) + a := depCtx(t, func(tools.Explanation) tools.Answer { return tools.Answer{Why: "answered no"} }) + a.stepDependencies() + if len(calls) != 0 { + t.Fatalf("a no ran %v", calls) + } + notes := strings.Join(a.notes, "\n") + for _, want := range []string{ + "what abcd uses it for:", + "install step", + "gitleaks not installed (answered no); continuing on the native secret scanner", + } { + if !strings.Contains(notes, want) { + t.Errorf("notes lack %q:\n%s", want, notes) + } + } + if len(a.changes) != 0 || len(a.writes) != 0 { + t.Errorf("a no reported a change or a write: %v %v", a.changes, a.writes) + } +} + +// TestDependencyWithoutAConfirmationNeverInstalls: a front door that supplies +// no confirmation (--yes, a caller that never asks) installs nothing. +func TestDependencyWithoutAConfirmationNeverInstalls(t *testing.T) { + var calls [][]string + fakeToolRun(t, &calls) + a := depCtx(t, nil) + a.stepDependencies() + if len(calls) != 0 { + t.Fatalf("installed with no confirmation: %v", calls) + } + if !strings.Contains(strings.Join(a.notes, "\n"), "continuing on the native secret scanner") { + t.Fatalf("the decline is silent: %v", a.notes) + } +} + +// TestDependencyYesRunsTheStepAndReportsIt is criterion 2's yes half at ahoy. +func TestDependencyYesRunsTheStepAndReportsIt(t *testing.T) { + var calls [][]string + fakeToolRun(t, &calls) + var shown tools.Explanation + a := depCtx(t, func(e tools.Explanation) tools.Answer { shown = e; return tools.Answer{Yes: true, Why: "typed yes"} }) + a.stepDependencies() + if shown.Tool != "gitleaks" { + t.Fatalf("the confirmation was not shown the explanation: %+v", shown) + } + if len(calls) != 2 || filepath.Base(calls[0][0]) != "brew" { + t.Fatalf("calls = %v, want the brew step and the verify", calls) + } + changes := strings.Join(a.changes, "\n") + if !strings.Contains(changes, "ran brew install gitleaks") || !strings.Contains(changes, "verified") { + t.Fatalf("the install is not reported as a change: %v (notes %v)", a.changes, a.notes) + } +} + +// TestEveryToolAhoyNamesIsRegistered is the build-time half of criterion 3: a +// tool ahoy names that the registry lacks fails here, naming it, before it can +// reach anyone as the generic explanation. DependencyTools and the gaps +// detectDependencies emits are the same set. +func TestEveryToolAhoyNamesIsRegistered(t *testing.T) { + emptyPath(t) + emitted := map[string]bool{} + for _, g := range detectDependencies(t.TempDir()) { + if g.Tool == nil { + t.Fatalf("dependency gap %s carries no explanation", g.ID) + } + emitted[g.Tool.Tool] = true + } + for _, n := range append(append([]string{}, DependencyTools...), "gh") { + if !tools.Known(n) { + t.Errorf("ahoy names %q, which the tool registry does not hold", n) + } + } + for _, n := range DependencyTools { + if !emitted[n] { + t.Errorf("DependencyTools names %q but no gap is emitted for it", n) + } + delete(emitted, n) + } + for n := range emitted { + t.Errorf("a gap names %q, which DependencyTools omits", n) + } +} diff --git a/internal/core/surface/sentences.go b/internal/core/surface/sentences.go index fe496ccff..1c7e1658c 100644 --- a/internal/core/surface/sentences.go +++ b/internal/core/surface/sentences.go @@ -29,7 +29,7 @@ var sentences = map[string]string{ "abcd ahoy doctor": "Report every install gap, user-scope state included: " + "Writes nothing; refuses any argument.", "abcd ahoy install": "Apply the install gaps the detection finds: " + - "Writes the .abcd/ scaffolding, the name-guard hooks, and the PATH entry; refuses a stale binary before any write.", + "Writes .abcd/, the name-guard hooks and the PATH entry, and installs a tool only on a yes; refuses a stale binary.", "abcd ahoy remote": "Enable GitHub secret scanning and push protection: " + "Writes nothing bare, only the settings and their mirror; refuses bare, naming `abcd ahoy --remote`.", "abcd ahoy remote apply": "Enable GitHub secret scanning and push protection on this repository: " + diff --git a/internal/surface/cli/ahoy_tool_confirm_test.go b/internal/surface/cli/ahoy_tool_confirm_test.go new file mode 100644 index 000000000..955d6e797 --- /dev/null +++ b/internal/surface/cli/ahoy_tool_confirm_test.go @@ -0,0 +1,90 @@ +package cli + +import ( + "bufio" + "bytes" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/core/ahoy" + "github.com/intentdriven/abcd/internal/core/tools" +) + +func gitleaksExplained() tools.Explanation { + return tools.Explain("gitleaks", tools.TranscriptScan) +} + +// TestToolConfirmAsksOnlyAtATerminal is itd-63's non-interactive rule at the +// CLI: a piped "y" (the `yes | abcd ahoy install` a host reaches for) answers +// the category questions, never the install of a program. Only an answer typed +// at a terminal, or the tool named with --install-tool, is a yes. +func TestToolConfirmAsksOnlyAtATerminal(t *testing.T) { + e := gitleaksExplained() + + var w bytes.Buffer + piped := &stdinPrompter{r: bufio.NewReader(strings.NewReader("y\n")), w: &w} + ans := toolConfirm(piped, nil, false, &w)(e) + if ans.Yes { + t.Fatal("a piped y installed a tool") + } + if !strings.Contains(ans.Why, "terminal") || !strings.Contains(ans.Why, "--install-tool gitleaks") { + t.Errorf("the decline does not say how to say yes: %q", ans.Why) + } + + w.Reset() + tty := &stdinPrompter{r: bufio.NewReader(strings.NewReader("y\n")), w: &w, tty: true} + ans = toolConfirm(tty, nil, false, &w)(e) + if !ans.Yes { + t.Fatalf("a y typed at a terminal was not a yes: %+v", ans) + } + asked := w.String() + for _, want := range []string{"what abcd uses it for:", "without it:", "brew install gitleaks", "[y/N]"} { + if !strings.Contains(asked, want) { + t.Errorf("the terminal question lacks %q:\n%s", want, asked) + } + } + + w.Reset() + tty = &stdinPrompter{r: bufio.NewReader(strings.NewReader("\n")), w: &w, tty: true} + if ans = toolConfirm(tty, nil, false, &w)(e); ans.Yes { + t.Fatal("a bare Enter at a terminal was a yes; the default is no") + } +} + +// TestToolConfirmNamedAndYes: --install-tool is the explicit answer a host's +// question tool relays; --yes never installs a tool, even at a terminal. +func TestToolConfirmNamedAndYes(t *testing.T) { + e := gitleaksExplained() + var w bytes.Buffer + tty := &stdinPrompter{r: bufio.NewReader(strings.NewReader("y\n")), w: &w, tty: true} + + if ans := toolConfirm(tty, map[string]bool{"gitleaks": true}, false, &w)(e); !ans.Yes { + t.Fatalf("a tool named with --install-tool was not a yes: %+v", ans) + } + ans := toolConfirm(tty, nil, true, &w)(e) + if ans.Yes || !strings.Contains(ans.Why, "--yes") { + t.Fatalf("--yes installed a tool or said nothing: %+v", ans) + } + if ans := toolConfirm(ahoy.RefusingPrompter{}, nil, false, &w)(e); ans.Yes { + t.Fatal("the refusing prompter installed a tool") + } +} + +// TestInstallToolRefusesAnUnknownName: an --install-tool name ahoy install +// does not check for is refused before anything runs, naming the ones it does. +// gh is in the registry but no install gap names it, so it is refused too: a +// flag that would be accepted and then do nothing is a silent no. +func TestInstallToolRefusesAnUnknownName(t *testing.T) { + t.Chdir(t.TempDir()) + for _, name := range []string{"frobnicate", "gh"} { + _, err := runCLIStdinErr(t, "", "ahoy", "install", "--install-tool", name, "--refuse-adopt") + if err == nil { + t.Fatalf("--install-tool %s was accepted", name) + } + for _, want := range append([]string{name}, ahoy.DependencyTools...) { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal lacks %q: %v", want, err) + } + } + } +} diff --git a/internal/surface/cli/cli.go b/internal/surface/cli/cli.go index 3aee32488..eedeb7c39 100644 --- a/internal/surface/cli/cli.go +++ b/internal/surface/cli/cli.go @@ -41,6 +41,7 @@ import ( "github.com/intentdriven/abcd/internal/core/rules" "github.com/intentdriven/abcd/internal/core/spec" "github.com/intentdriven/abcd/internal/core/surface" + "github.com/intentdriven/abcd/internal/core/tools" "github.com/intentdriven/abcd/internal/core/update" "github.com/intentdriven/abcd/internal/fsutil" "github.com/intentdriven/abcd/internal/gitutil" @@ -2964,6 +2965,7 @@ func newAhoyCommand(asJSON *bool) *cobra.Command { docsTarget string oracleBackend string scanDeep string + installTools []string ) installCmd := &cobra.Command{ Use: "install", @@ -2977,7 +2979,13 @@ func newAhoyCommand(asJSON *bool) *cobra.Command { if err != nil { return err } - res, err := ahoy.Install(cwd, opts, newPrompter(cmd)) + named, err := installToolNames(installTools) + if err != nil { + return err + } + p := newPrompter(cmd) + opts.ConfirmTool = toolConfirm(p, named, yes, cmd.ErrOrStderr()) + res, err := ahoy.Install(cwd, opts, p) if err != nil { return err } @@ -3030,6 +3038,7 @@ func newAhoyCommand(asJSON *bool) *cobra.Command { installCmd.Flags().StringVar(&docsTarget, "docs-target", "", "which conventions file carries the managed block, which names abcd: claude_md | agents_md | both | skip (default skip)") installCmd.Flags().StringVar(&oracleBackend, "oracle-backend", "", "oracle backend: host-delegated | native | cli | api | mcp") installCmd.Flags().StringVar(&scanDeep, "scan-deep", "", "enable deep scan: true | false") + installCmd.Flags().StringSliceVar(&installTools, "install-tool", nil, "answer yes to installing this missing tool (repeatable): the answer a host's question tool relays; without it a tool is installed only on an answer typed at a terminal, never on the approve-everything flag, a piped answer or CI") ahoyCmd.AddCommand(installCmd) // uninstall @@ -3318,6 +3327,57 @@ func optionalSkipReason(id string) string { return "" } +// installToolNames validates the --install-tool names against the tools +// `ahoy install` checks for, refusing any other with the names it accepts. +func installToolNames(names []string) (map[string]bool, error) { + accepted := map[string]bool{} + for _, n := range ahoy.DependencyTools { + accepted[n] = true + } + named := map[string]bool{} + for _, n := range names { + n = strings.TrimSpace(n) + if !accepted[n] { + return nil, &exitError{Code: 2, Msg: fmt.Sprintf("abcd ahoy install: --install-tool %q is not a tool ahoy install checks for; it checks for %s (nothing read, nothing written)", + termsafe.Sanitize(n), strings.Join(ahoy.DependencyTools, ", "))} + } + named[n] = true + } + return named, nil +} + +// toolConfirm is the CLI's answer to the explain-then-install question +// (itd-63). It is deliberately narrower than the category prompter: a piped +// answer approves categories (iss-167), but installing a program is asked only +// of a person at a terminal, or answered by naming the tool with +// --install-tool, which is how a host relays the answer its own question tool +// got. --yes never installs a tool. Every no carries the way to say yes. +func toolConfirm(p ahoy.Prompter, named map[string]bool, yes bool, w io.Writer) tools.Confirm { + return func(e tools.Explanation) tools.Answer { + if named[e.Tool] { + return tools.Answer{Yes: true, Why: "named with --install-tool"} + } + if yes { + return tools.Answer{Why: "--yes never installs a tool; name it with --install-tool " + e.Tool + ", or run without --yes at a terminal"} + } + sp, ok := p.(*stdinPrompter) + if !ok || !sp.tty { + return tools.Answer{Why: "no terminal to ask at: abcd installs a tool only on an answer typed at a terminal, or with --install-tool " + e.Tool} + } + for _, line := range e.Lines() { + fmt.Fprintln(w, termsafe.Sanitize(line)) + } + fmt.Fprintf(w, "Install %s now by running %s? [y/N] ", e.Tool, e.StepText()) + line, _ := sp.r.ReadString('\n') + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + fmt.Fprintf(w, "running %s; a package manager can take a few minutes\n", e.StepText()) + return tools.Answer{Yes: true, Why: "answered yes at the terminal"} + } + return tools.Answer{Why: "answered no at the terminal"} + } +} + // newPrompter returns the stdin-reading prompter. On a terminal it is the // interactive path, unchanged. When stdin is NOT a terminal the same prompter // reads the piped answers (iss-167): `yes | abcd ahoy install` is what a host From d2a71c761e4ef08a08117dffd06ff5f1814b8db3 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:02:36 +0100 Subject: [PATCH 07/60] feat(ahoy): a named tool answers the dependency question `ahoy install --install-tool ` is the answer a host relays from its own question tool, so it has to reach the install step with stdin closed. It now approves the dependency category without asking its question (ApproveDependency); every other category is asked or pre-answered as before. End-to-end tests drive both halves through the CLI with no package manager on PATH, so no test can ever install. Assisted-by: Claude:claude-opus-5-5 --- commands/ahoy.md | 6 +- internal/core/ahoy/ahoy.go | 5 ++ internal/core/ahoy/apply.go | 6 ++ internal/core/ahoy/tools_route_test.go | 29 +++++++ .../surface/cli/ahoy_tool_confirm_test.go | 80 +++++++++++++++++++ internal/surface/cli/cli.go | 1 + 6 files changed, 125 insertions(+), 2 deletions(-) diff --git a/commands/ahoy.md b/commands/ahoy.md index 9688fc6a4..6766b2614 100644 --- a/commands/ahoy.md +++ b/commands/ahoy.md @@ -161,8 +161,10 @@ run "${CLAUDE_PLUGIN_ROOT}/abcd" ahoy install --install-tool --json ``` -`--install-tool` is the relayed yes, for the named tool only; never pass it -without the user's answer. The result's `changes` reports what ran and whether +`--install-tool` is the relayed yes, for the named tool only, and it also +answers the `dependency` category's question, which is then not asked (so a +piped answer stream has one question fewer); never pass it without the user's +answer. The result's `changes` reports what ran and whether its verify passed; a no, a failed step or a missing package manager is a `notes` line ending in what the capability continues on (`continuing on the native secret scanner`). `--yes` never installs a tool, and a run with `CI` set diff --git a/internal/core/ahoy/ahoy.go b/internal/core/ahoy/ahoy.go index 8c2de3252..3d1751da3 100644 --- a/internal/core/ahoy/ahoy.go +++ b/internal/core/ahoy/ahoy.go @@ -161,6 +161,11 @@ type InstallOptions struct { // --install-tool. Nil is a no for every tool, so a caller that asks nothing // installs nothing. The category approval never stands in for it. ConfirmTool tools.Confirm + // ApproveDependency answers the dependency category's question yes without + // asking it: the front door sets it when the person named a tool to + // install (--install-tool), which is that answer given in advance. Every + // other category is still asked, or pre-answered, as before. + ApproveDependency bool } // InstallResult is the outcome of Install. diff --git a/internal/core/ahoy/apply.go b/internal/core/ahoy/apply.go index 1b8798bdd..c2b4bb95c 100644 --- a/internal/core/ahoy/apply.go +++ b/internal/core/ahoy/apply.go @@ -1746,11 +1746,17 @@ func resolveApproval(gaps []Gap, opts InstallOptions, p Prompter) (map[GapCatego } default: for _, c := range presentInPromptOrder(present) { + if c == Dependency && opts.ApproveDependency { + continue // answered by the named tool; approved below + } if p.Confirm("Apply " + string(c) + " changes?") { approved[c] = true } } } + if opts.ApproveDependency && present[Dependency] { + approved[Dependency] = true + } var declined []string for c := range present { if !approved[c] { diff --git a/internal/core/ahoy/tools_route_test.go b/internal/core/ahoy/tools_route_test.go index 1dacde4ae..fceb517e1 100644 --- a/internal/core/ahoy/tools_route_test.go +++ b/internal/core/ahoy/tools_route_test.go @@ -204,3 +204,32 @@ func TestEveryToolAhoyNamesIsRegistered(t *testing.T) { t.Errorf("a gap names %q, which DependencyTools omits", n) } } + +// TestNamedToolApprovesTheDependencyCategory: naming a tool to install is the +// answer to the dependency question, so a host relaying it with no stdin +// reaches the step; every other category is still asked (and here declined). +func TestNamedToolApprovesTheDependencyCategory(t *testing.T) { + gaps := []Gap{ + {ID: "deps.gitleaks_missing", Category: Dependency, Resolvable: true}, + {ID: "skeleton.config_missing", Category: SafeAutocreate, Resolvable: true}, + } + p := &recordingPrompter{} + approved, declined := resolveApproval(gaps, InstallOptions{ApproveDependency: true}, p) + asked := p.asked + if !approved[Dependency] { + t.Fatalf("a named tool did not approve the dependency category (declined %v)", declined) + } + for _, q := range asked { + if strings.Contains(q, string(Dependency)) { + t.Errorf("the dependency question was still asked: %v", asked) + } + } + if approved[SafeAutocreate] || len(asked) != 1 { + t.Errorf("the other categories were not left to their own answers: approved %v asked %v", approved, asked) + } + + approved, _ = resolveApproval(gaps, InstallOptions{ApproveDependency: true, ApprovedCategories: map[GapCategory]bool{}}, &recordingPrompter{}) + if !approved[Dependency] { + t.Fatal("an explicit category subset dropped the named tool's approval") + } +} diff --git a/internal/surface/cli/ahoy_tool_confirm_test.go b/internal/surface/cli/ahoy_tool_confirm_test.go index 955d6e797..ff9075e1b 100644 --- a/internal/surface/cli/ahoy_tool_confirm_test.go +++ b/internal/surface/cli/ahoy_tool_confirm_test.go @@ -3,6 +3,10 @@ package cli import ( "bufio" "bytes" + "encoding/json" + "os" + "os/exec" + "path/filepath" "strings" "testing" @@ -88,3 +92,79 @@ func TestInstallToolRefusesAnUnknownName(t *testing.T) { } } } + +// toolFreePath leaves git on PATH (the install reads the checkout) and nothing +// else, so neither gitleaks nor a package manager is found and no install +// step can ever run from this test. +func toolFreePath(t *testing.T) { + t.Helper() + gitBin, err := exec.LookPath("git") + if err != nil { + t.Skip("git not on PATH") + } + dir := t.TempDir() + if err := os.Symlink(gitBin, filepath.Join(dir, "git")); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir) + t.Setenv("CI", "") +} + +func installNotes(t *testing.T, out []byte) (notes, declined []string) { + t.Helper() + var res struct { + Notes []string `json:"notes"` + DeclinedCategories []string `json:"declined_categories"` + } + if err := json.Unmarshal(out, &res); err != nil { + t.Fatalf("install output not JSON: %v\n%s", err, out) + } + return res.Notes, res.DeclinedCategories +} + +// TestAhoyInstallPipedYesExplainsAndKeepsTheNativeScanner is the end-to-end +// no: `yes | abcd ahoy install` approves every category, and the missing +// gitleaks is explained and left uninstalled, loudly, on the native scanner. +func TestAhoyInstallPipedYesExplainsAndKeepsTheNativeScanner(t *testing.T) { + hermeticRepo(t) + toolFreePath(t) + out, errOut, err := runCLIPipedStdinSplit(t, strings.Repeat("y\n", 12), "ahoy", "install", "--allow-stale-binary", "--json") + if err != nil { + t.Fatalf("install exited non-zero: %v\n%s\n%s", err, out, errOut) + } + notes, _ := installNotes(t, out) + joined := strings.Join(notes, "\n") + for _, want := range []string{ + "dependency: gitleaks — optional for", + "install step (Homebrew): brew install gitleaks", + "gitleaks not installed (no terminal to ask at", + "continuing on the native secret scanner", + } { + if !strings.Contains(joined, want) { + t.Errorf("notes lack %q:\n%s", want, joined) + } + } +} + +// TestAhoyInstallNamedToolReachesTheStep is the host's relayed yes end to +// end: with stdin closed, --install-tool answers the dependency question and +// the install is attempted; here the package manager is absent, so the +// result says so and nothing ran. +func TestAhoyInstallNamedToolReachesTheStep(t *testing.T) { + hermeticRepo(t) + toolFreePath(t) + out, errOut, err := runCLIPipedStdinSplit(t, "", "ahoy", "install", "--adopt", "--allow-stale-binary", "--install-tool", "gitleaks", "--json") + if err != nil { + t.Fatalf("install exited non-zero: %v\n%s\n%s", err, out, errOut) + } + notes, declined := installNotes(t, out) + for _, c := range declined { + if c == "dependency" { + t.Fatalf("--install-tool did not answer the dependency question: declined %v", declined) + } + } + joined := strings.Join(notes, "\n") + if !strings.Contains(joined, "Homebrew (brew) is not on PATH") || !strings.Contains(joined, "continuing on the native secret scanner") { + t.Fatalf("the named install did not reach the step, or was silent:\n%s", joined) + } +} diff --git a/internal/surface/cli/cli.go b/internal/surface/cli/cli.go index eedeb7c39..867ca139d 100644 --- a/internal/surface/cli/cli.go +++ b/internal/surface/cli/cli.go @@ -2985,6 +2985,7 @@ func newAhoyCommand(asJSON *bool) *cobra.Command { } p := newPrompter(cmd) opts.ConfirmTool = toolConfirm(p, named, yes, cmd.ErrOrStderr()) + opts.ApproveDependency = len(named) > 0 res, err := ahoy.Install(cwd, opts, p) if err != nil { return err From 22e6edbd1fde4276840cae069cdab278fec19bf7 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:03:09 +0100 Subject: [PATCH 08/60] test(cli): the named-tool yes holds with nothing typed The named case answered through a terminal that also typed y, so a confirmation that ignored --install-tool still passed. It now answers off a terminal with empty input, where only the flag can be the yes; watched failing with the named branch disabled on a scratch copy. Assisted-by: Claude:claude-opus-5-5 --- internal/surface/cli/ahoy_tool_confirm_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/surface/cli/ahoy_tool_confirm_test.go b/internal/surface/cli/ahoy_tool_confirm_test.go index ff9075e1b..690a7669d 100644 --- a/internal/surface/cli/ahoy_tool_confirm_test.go +++ b/internal/surface/cli/ahoy_tool_confirm_test.go @@ -62,7 +62,10 @@ func TestToolConfirmNamedAndYes(t *testing.T) { var w bytes.Buffer tty := &stdinPrompter{r: bufio.NewReader(strings.NewReader("y\n")), w: &w, tty: true} - if ans := toolConfirm(tty, map[string]bool{"gitleaks": true}, false, &w)(e); !ans.Yes { + // Off a terminal, and with nothing typed, the named tool is still a yes: + // the flag is the answer. + piped := &stdinPrompter{r: bufio.NewReader(strings.NewReader("")), w: &w} + if ans := toolConfirm(piped, map[string]bool{"gitleaks": true}, false, &w)(e); !ans.Yes { t.Fatalf("a tool named with --install-tool was not a yes: %+v", ans) } ans := toolConfirm(tty, nil, true, &w)(e) From 90aa2e88690a54808d4db0c36b43506d16df00e6 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:03:20 +0100 Subject: [PATCH 09/60] =?UTF-8?q?chore:=20resolve=20iss-2609261447331434?= =?UTF-8?q?=20=E2=80=94=20ahoy=20stops=20offering=20trufflehog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves: iss-2609261447331434 Assisted-by: Claude:claude-opus-5-5 --- ...ffers-trufflehog-as-an-optional-dependency-the-deps.md | 8 ++++++++ 1 file changed, 8 insertions(+) rename .abcd/work/issues/{open => resolved}/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md (61%) diff --git a/.abcd/work/issues/open/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md b/.abcd/work/issues/resolved/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md similarity index 61% rename from .abcd/work/issues/open/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md rename to .abcd/work/issues/resolved/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md index b2c7981ed..0308e7bde 100644 --- a/.abcd/work/issues/open/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md +++ b/.abcd/work/issues/resolved/iss-2609261447331434-ahoy-offers-trufflehog-as-an-optional-dependency-the-deps.md @@ -9,6 +9,14 @@ found_during: "autonomous run A resumed 2026-09-25" origin: researcher-authored production_mode: hand-written found_at: "internal/core/ahoy/detect.go" +resolution: "ahoy no longer emits the deps.trufflehog_missing gap: nothing in abcd runs trufflehog, so the explain-then-install mode would have offered to install a program that does nothing for the person. A test pins that no dependency gap names trufflehog." +impact: fix +resolved_by: + commit: "920afe82" --- ahoy offers trufflehog as an optional dependency (the deps.trufflehog_missing gap, 'trufflehog enables deep secret scanning when scan.deep=true', fix hint 'brew install trufflehog') although nothing in abcd runs trufflehog: no adapter exists and scan.deep is read by no scanner. The gap asks a person to install a tool abcd never uses, which the explain-then-install mode (itd-63) would turn into an executed install of a program that does nothing for them. + +## Grounds + +- pursued: we expect no ahoy run to offer trufflehog while no adapter runs it; shown wrong if a trufflehog gap or install offer reappears without a wired trufflehog adapter From d3f1f599c57358e3d434800662175993a3cb7c24 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:03:25 +0100 Subject: [PATCH 10/60] feat(capture,intent): match a new capture or draft against the record before it is written `capture` and the quoted-text `intent` create now take a match request: under the writer's existing lock (the ledger lock; the intent mint lock), the new text is compared with every open and resolved issue's body and every intent's title and press release, as far as match.fields names them, through the canonical overlap primitive. Each candidate above the threshold is written onto the new record as `duplicates:` or `refines:` (an inline id list, at most three links), and the result carries the outcome: the matches, the near misses below the threshold with their scores, or why nothing was compared (a text below the declared minimum of terms, or a candidate set that could not be read). The match never refuses and never drops a write: a failure to gather the candidates, or a link the schema would refuse, files the record unlinked and says so on the outcome. Removing a link leaves an ordinary record. The two link keys are KNOWN issue properties (issueschema), id lists validated like every other, read back onto Issue, and record_schema resolves their targets like any other cross-reference. The intent create takes its candidates through a Matcher the ledger fills, because the ledger reads intents and the intent store does not read the ledger. Callers with their own matching (the inbox drain, promote) pass none. Assisted-by: Claude:claude-opus-5-5 --- internal/core/capture/capture.go | 19 ++ internal/core/capture/match.go | 119 ++++++++++ internal/core/capture/match_test.go | 263 +++++++++++++++++++++++ internal/core/capture/validate.go | 7 +- internal/core/capture/workflow.go | 10 +- internal/core/intent/create.go | 79 +++++-- internal/core/intent/match.go | 126 +++++++++++ internal/core/intent/match_test.go | 139 ++++++++++++ internal/core/intent/ready_test.go | 2 +- internal/core/issueschema/issueschema.go | 6 + internal/core/lint/matchlinks_test.go | 27 +++ internal/core/lint/schema.go | 4 +- internal/core/record/match/match.go | 19 ++ 13 files changed, 795 insertions(+), 25 deletions(-) create mode 100644 internal/core/capture/match.go create mode 100644 internal/core/capture/match_test.go create mode 100644 internal/core/intent/match.go create mode 100644 internal/core/intent/match_test.go create mode 100644 internal/core/lint/matchlinks_test.go diff --git a/internal/core/capture/capture.go b/internal/core/capture/capture.go index 88ff739db..2128dd57d 100644 --- a/internal/core/capture/capture.go +++ b/internal/core/capture/capture.go @@ -20,6 +20,7 @@ import ( "regexp" "github.com/intentdriven/abcd/internal/core/issueschema" + "github.com/intentdriven/abcd/internal/core/record/match" "github.com/intentdriven/abcd/internal/core/recordid" "github.com/intentdriven/abcd/internal/core/relink" ) @@ -108,6 +109,12 @@ type Issue struct { RelatedSpecs []string `json:"related_specs,omitempty"` RelatedIssues []string `json:"related_issues,omitempty"` BlockedBy []string `json:"blocked_by,omitempty"` // iss-N dependency edges + // Duplicates and Refines are the typed links the filing-time match writes + // (itd-2609212137116617): the iss-N or itd-N this record is a near-identical + // double of, or a narrower case of. A person confirms a link by leaving it + // and removes it by deleting the line; a record without either is ordinary. + Duplicates []string `json:"duplicates,omitempty"` + Refines []string `json:"refines,omitempty"` // Grounds is the record's recorded conjectures, in the order they were // written: one `: ` value in the shared core/grounds vocabulary // per grounds-bearing act. Appended by promote, resolve and wontfix; never by @@ -163,6 +170,14 @@ type CaptureRequest struct { // it is derived from which command ran, and a capture is researcher-authored // by construction. ProductionMode string + // Match, when non-nil, matches the text against every open and resolved + // issue and every intent before the record is written, under the ledger + // lock, and writes a `duplicates:` or `refines:` link naming each likely + // double (itd-2609212137116617). It never refuses the capture: a match that + // cannot run says why on the result and the record is filed without it. + // nil files the record unmatched, as a caller with its own matching (the + // inbox drain, the consistency pass) does. + Match *match.Config } // CaptureResult is the outcome of a successful Capture. The timestamp-numeric @@ -189,6 +204,10 @@ type CaptureResult struct { // record to the repository it is filed into — the shape every misfiled // record of iss-2609120511058115 had (iss-2609231156260287). NoLocation bool `json:"no_location,omitempty"` + // Match is the filing-time match's outcome when the request asked for one: + // the links written, the near misses below the threshold with their scores, + // or why nothing was compared. + Match *match.Outcome `json:"match,omitempty"` } // ResolveRequest moves an open issue to resolved/. diff --git a/internal/core/capture/match.go b/internal/core/capture/match.go new file mode 100644 index 000000000..f66490146 --- /dev/null +++ b/internal/core/capture/match.go @@ -0,0 +1,119 @@ +package capture + +import ( + "fmt" + "regexp" + "strings" + + "github.com/intentdriven/abcd/internal/core/intent" + "github.com/intentdriven/abcd/internal/core/record/match" +) + +// match.go is the ledger's half of the filing-time match +// (itd-2609212137116617): the candidate set a new capture or draft is compared +// with — every open and resolved issue and every intent, through the fields the +// configuration names — and the typed links a capture writes. The score is +// internal/core/record/match's, the one term-overlap primitive. The ledger +// gathers the intents too, because it already reads the intent store and the +// intent store does not read the ledger; the intent create takes this set +// through its Matcher. + +// reLinkID is the shape of a typed link's target: an issue or an intent, the +// two families the match compares with. +var reLinkID = regexp.MustCompile(`^(iss|itd)-[0-9]+$`) + +// linkFields are the typed-link keys a record may carry, in the order a +// capture writes them. +var linkFields = []match.Relation{match.Duplicates, match.Refines} + +// MatchCandidates is the candidate set for a filing in the checkout at +// repoRoot, for a caller outside this package: the quoted-text intent create. +func MatchCandidates(repoRoot string, cfg match.Config) ([]match.Candidate, error) { + rr, issuesRoot, err := resolveRoots(repoRoot, "") + if err != nil { + return nil, err + } + return matchCandidates(rr, issuesRoot, cfg) +} + +// matchCandidates reads every open and resolved issue's body and every +// intent's title and press release, as far as cfg compares them. A wontfix +// record is not a candidate: it names work nobody is doing. A ledger record +// the reader skips is not one either, since its text cannot be trusted; the +// ledger's own surfaces already report it. +func matchCandidates(repoRoot, issuesRoot string, cfg match.Config) ([]match.Candidate, error) { + var out []match.Candidate + if cfg.Compares(match.FieldIssueBody) { + for _, st := range []State{StateOpen, StateResolved} { + issues, _ := scanLedger(issuesRoot, st) + for _, iss := range issues { + out = append(out, match.Candidate{ID: iss.ID, Text: iss.Body}) + } + } + } + if cfg.Compares(match.FieldIntentTitle) || cfg.Compares(match.FieldIntentPressRelease) { + texts, err := intent.MatchTexts(repoRoot) + if err != nil { + return nil, err + } + for _, t := range texts { + var parts []string + if cfg.Compares(match.FieldIntentTitle) { + parts = append(parts, t.Title) + } + if cfg.Compares(match.FieldIntentPressRelease) { + parts = append(parts, t.PressRelease) + } + out = append(out, match.Candidate{ID: t.ID, Text: strings.Join(parts, "\n")}) + } + } + return out, nil +} + +// matchAndLink runs the match for a capture and writes its links into the +// record's rendered content, validating the frontmatter they join. It runs +// under the ledger lock, before the write. It never fails the capture: a +// candidate set that cannot be read, or a link the schema would refuse, comes +// back as an outcome saying why, with the content unlinked. +func matchAndLink(repoRoot, issuesRoot string, cfg match.Config, text, content string, fm map[string]any) (string, *match.Outcome) { + if match.Short(text) { + o := match.Rank(text, nil, cfg.Threshold) + return content, &o + } + cands, err := matchCandidates(repoRoot, issuesRoot, cfg) + if err != nil { + o := match.Unread(cfg.Threshold, err) + return content, &o + } + o := match.Rank(text, cands, cfg.Threshold) + links := o.Links() + if len(links) == 0 { + return content, &o + } + linked := content + withLinks := make(map[string]any, len(fm)+len(links)) + for k, v := range fm { + withLinks[k] = v + } + for _, rel := range linkFields { + ids := links[rel] + if len(ids) == 0 { + continue + } + withLinks[string(rel)] = ids + if linked, err = setListField(linked, string(rel), ids); err != nil { + break + } + } + if err == nil { + err = validateStrict(withLinks) + } + if err != nil { + o.Skipped = fmt.Sprintf("the links could not be written (%v), so the record is filed unlinked", err) + for i := range o.Matches { + o.Matches[i].Linked = false + } + return content, &o + } + return linked, &o +} diff --git a/internal/core/capture/match_test.go b/internal/core/capture/match_test.go new file mode 100644 index 000000000..c8f3d15c6 --- /dev/null +++ b/internal/core/capture/match_test.go @@ -0,0 +1,263 @@ +package capture + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/core/intent" + "github.com/intentdriven/abcd/internal/core/record/match" +) + +// The planted double: a finding, and the same finding filed again in other +// words. The fillers are records about other things, so the corpus has a +// vocabulary to weigh terms against. +const ( + plantedFinding = "The capture ledger reader silently skips a record whose frontmatter carries " + + "a duplicated key, so the finding disappears from every listing without a warning." + plantedDouble = "Capture ledger reader silently skips any record whose frontmatter carries a " + + "duplicated key: the finding disappears from every listing, and no warning is printed." + matchFiller1 = "The site builder renders a stale anchor for a heading renamed since the last build." + matchFiller2 = "The history store drops a transcript that exceeds its byte budget without saying so." + // A broad intent the narrow capture below refines. + broadTitle = "Record readers agree with the lint gate" + broadPress = "Record readers disagree with the lint gate. The capture ledger reader skips " + + "records with duplicated frontmatter keys, the intent loader fails closed on a missing id, " + + "the spec store tolerates unknown properties, the site builder renders stale anchors, " + + "the memory store truncates pages, and the history store drops transcripts over its budget." + narrowCapture = "The capture ledger reader skips records with duplicated frontmatter keys " + + "while the lint gate reports them." +) + +func captureText(t *testing.T, repo, ir, text string, m *match.Config) CaptureResult { + t.Helper() + res, err := Capture(CaptureRequest{ + RepoRoot: repo, IssuesRoot: ir, Text: text, Severity: SeverityMinor, + Category: "bug", Source: "user-observation", FoundDuring: "t", Match: m, + }) + if err != nil { + t.Fatalf("capture refused: %v", err) + } + return res +} + +func plantIntent(t *testing.T, repo, bucket, id, title, press string) { + t.Helper() + writeFile(t, filepath.Join(repo, ".abcd/development/intents", bucket, id+"-planted.md"), + "---\nid: "+id+"\nslug: planted\nspec_id: null\nkind: null\n---\n\n# "+title+ + "\n\n## Press Release\n\n> "+press+"\n\n## Why This Matters\n\nUnrelated words about budgets.\n") +} + +func readRecord(t *testing.T, repo, rel string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(repo, rel)) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func bundled() *match.Config { c := match.Bundled(); return &c } + +// Criterion 1: a capture that overlaps an existing issue above the threshold +// is written with a typed link naming it, and the result carries the match. +func TestCaptureLinksAPlantedDouble(t *testing.T) { + repo, ir := ledger(t) + first := captureText(t, repo, ir, plantedFinding, nil) + captureText(t, repo, ir, matchFiller1, nil) + captureText(t, repo, ir, matchFiller2, nil) + + res := captureText(t, repo, ir, plantedDouble, bundled()) + if res.Match == nil || len(res.Match.Matches) == 0 { + t.Fatalf("no match reported: %+v", res.Match) + } + m := res.Match.Matches[0] + if m.ID != first.ID || m.Relation != match.Duplicates || !m.Linked { + t.Fatalf("match = %+v, want %s linked as duplicates", m, first.ID) + } + content := readRecord(t, repo, res.Path) + if !strings.Contains(content, "\nduplicates: ["+first.ID+"]\n") { + t.Fatalf("the record carries no duplicates link naming %s:\n%s", first.ID, content) + } + // The link reads back through the ledger reader. + list, err := List(ListRequest{RepoRoot: repo, IssuesRoot: ir, State: StateOpen}) + if err != nil { + t.Fatal(err) + } + for _, iss := range list.Issues { + if iss.ID == res.ID { + if len(iss.Duplicates) != 1 || iss.Duplicates[0] != first.ID { + t.Fatalf("read back duplicates = %v", iss.Duplicates) + } + return + } + } + t.Fatalf("the linked record %s is not listed (skipped: %+v)", res.ID, list.Skipped) +} + +// The candidates include every intent's title and press release, and a +// narrower capture of a broader record is written as `refines`. +func TestCaptureRefinesABroaderIntent(t *testing.T) { + repo, ir := ledger(t) + captureText(t, repo, ir, matchFiller1, nil) + captureText(t, repo, ir, matchFiller2, nil) + plantIntent(t, repo, "planned", "itd-9", broadTitle, broadPress) + + res := captureText(t, repo, ir, narrowCapture, bundled()) + if res.Match == nil || len(res.Match.Matches) != 1 || res.Match.Matches[0].ID != "itd-9" || + res.Match.Matches[0].Relation != match.Refines { + t.Fatalf("match = %+v, want itd-9 as refines", res.Match) + } + if content := readRecord(t, repo, res.Path); !strings.Contains(content, "\nrefines: [itd-9]\n") { + t.Fatalf("no refines link:\n%s", content) + } +} + +// Criterion 4: below the threshold nothing is written and the near misses +// are listed with their scores. +func TestCaptureBelowTheThresholdWritesNothingAndListsNearMisses(t *testing.T) { + repo, ir := ledger(t) + captureText(t, repo, ir, plantedFinding, nil) + captureText(t, repo, ir, matchFiller1, nil) + cfg := match.Bundled() + cfg.Threshold = 0.99 + res := captureText(t, repo, ir, narrowCapture, &cfg) + if res.Match == nil || len(res.Match.Matches) != 0 { + t.Fatalf("matched above a 0.99 threshold: %+v", res.Match) + } + if len(res.Match.NearMisses) == 0 || res.Match.NearMisses[0].Score <= 0 { + t.Fatalf("near misses = %+v, want the finding with its score", res.Match.NearMisses) + } + content := readRecord(t, repo, res.Path) + if regexp.MustCompile(`(?m)^(duplicates|refines):`).MatchString(content) { + t.Fatalf("a link was written below the threshold:\n%s", content) + } +} + +// The compared fields are configuration: with issue bodies left out, the +// planted double is not a candidate. +func TestCaptureComparesOnlyTheConfiguredFields(t *testing.T) { + repo, ir := ledger(t) + captureText(t, repo, ir, plantedFinding, nil) + cfg := match.Bundled() + cfg.Fields = []string{match.FieldIntentTitle, match.FieldIntentPressRelease} + res := captureText(t, repo, ir, plantedDouble, &cfg) + if res.Match == nil || len(res.Match.Matches)+len(res.Match.NearMisses) != 0 { + t.Fatalf("an issue was compared though issue.body is not a configured field: %+v", res.Match) + } +} + +// Criterion 3: a match that cannot run never refuses the capture; the record +// is filed and the result says why nothing was compared. A short text is +// filed without matching and says so. +func TestCaptureIsNeverRefusedByTheMatch(t *testing.T) { + repo, ir := ledger(t) + captureText(t, repo, ir, plantedFinding, nil) + // An intent store no reader can load: a record whose id is malformed. + writeFile(t, filepath.Join(repo, ".abcd/development/intents/drafts/itd-1-broken.md"), "---\nid: nonsense\n---\n") + res := captureText(t, repo, ir, plantedDouble, bundled()) + if res.Match == nil || res.Match.Skipped == "" || len(res.Match.Matches) != 0 { + t.Fatalf("an unreadable record set did not report a skipped match: %+v", res.Match) + } + if _, err := os.Stat(filepath.Join(repo, res.Path)); err != nil { + t.Fatalf("the capture was not written: %v", err) + } + + short := captureText(t, repo, ir, "Typo in the ledger README.", bundled()) + if short.Match == nil || !strings.Contains(short.Match.Skipped, "minimum") { + t.Fatalf("a short capture did not say it was filed without matching: %+v", short.Match) + } +} + +// Criterion 3, second half: removing the link leaves an ordinary record, one +// every reader takes exactly as it takes a record that never had one. +func TestRemovingTheLinkLeavesAnOrdinaryRecord(t *testing.T) { + repo, ir := ledger(t) + captureText(t, repo, ir, plantedFinding, nil) + captureText(t, repo, ir, matchFiller1, nil) + res := captureText(t, repo, ir, plantedDouble, bundled()) + p := filepath.Join(repo, res.Path) + content := readRecord(t, repo, res.Path) + stripped := regexp.MustCompile(`(?m)^duplicates: .*\n`).ReplaceAllString(content, "") + if stripped == content { + t.Fatalf("no link to remove:\n%s", content) + } + if err := os.WriteFile(p, []byte(stripped), 0o644); err != nil { + t.Fatal(err) + } + list, err := List(ListRequest{RepoRoot: repo, IssuesRoot: ir, State: StateOpen}) + if err != nil { + t.Fatal(err) + } + if len(list.Skipped) != 0 { + t.Fatalf("the unlinked record is skipped: %+v", list.Skipped) + } + for _, iss := range list.Issues { + if iss.ID == res.ID && len(iss.Duplicates)+len(iss.Refines) == 0 { + return + } + } + t.Fatalf("%s not read back as an ordinary record", res.ID) +} + +// Without a Match the capture is exactly the one it always was. +func TestCaptureWithoutAMatchReportsNone(t *testing.T) { + repo, ir := ledger(t) + captureText(t, repo, ir, plantedFinding, nil) + res := captureText(t, repo, ir, plantedDouble, nil) + if res.Match != nil { + t.Fatalf("an unrequested match ran: %+v", res.Match) + } + if strings.Contains(readRecord(t, repo, res.Path), "duplicates:") { + t.Fatal("an unrequested match wrote a link") + } +} + +// A link names a record by its id: a value of any other shape is refused by +// the reader, like every other id list. +func TestALinkValueIsARecordID(t *testing.T) { + fm := map[string]any{ + "schema_version": 1, "id": "iss-1", "slug": "a", "severity": "minor", + "category": "bug", "source": "user-observation", "found_during": "t", + } + for _, key := range []string{"duplicates", "refines"} { + for _, v := range []string{"iss-2", "itd-3"} { + fm[key] = []string{v} + if err := validateStrict(fm); err != nil { + t.Fatalf("%s: [%s] refused: %v", key, v, err) + } + } + fm[key] = []string{"spc-4"} + if err := validateStrict(fm); err == nil { + t.Fatalf("%s: [spc-4] admitted", key) + } + delete(fm, key) + } +} + +// Criterion 2 end to end: the quoted-text intent create, with the ledger's +// candidate set, links a draft that doubles an existing intent's title and +// press release. +func TestIntentCreateMatchesTheRecordThroughTheLedger(t *testing.T) { + repo, ir := ledger(t) + captureText(t, repo, ir, matchFiller1, nil) + captureText(t, repo, ir, matchFiller2, nil) + plantIntent(t, repo, "shipped", "itd-9", broadTitle, broadPress) + cfg := match.Bundled() + m := &intent.Matcher{Threshold: cfg.Threshold, Candidates: func() ([]match.Candidate, error) { + return MatchCandidates(repo, cfg) + }} + c, err := intent.CreateFromTextMatched(repo, broadPress, intent.TextOptions{Title: broadTitle}, m) + if err != nil { + t.Fatal(err) + } + if c.Match == nil || len(c.Match.Matches) == 0 || c.Match.Matches[0].ID != "itd-9" || + c.Match.Matches[0].Relation != match.Duplicates { + t.Fatalf("match = %+v, want itd-9 as duplicates", c.Match) + } + if content := readRecord(t, repo, c.Path); !strings.Contains(content, "\nduplicates: [itd-9]\n") { + t.Fatalf("no link on the draft:\n%s", content) + } +} diff --git a/internal/core/capture/validate.go b/internal/core/capture/validate.go index 5d96c1247..57596cf50 100644 --- a/internal/core/capture/validate.go +++ b/internal/core/capture/validate.go @@ -20,7 +20,7 @@ var knownFields = issueschema.Known // uniqueItemsFields are the array properties issue.schema.json flags // uniqueItems:true. -var uniqueItemsFields = []string{"related_intents", "related_specs", "related_issues", "synthesis_clusters", "blocked_by"} +var uniqueItemsFields = []string{"related_intents", "related_specs", "related_issues", "synthesis_clusters", "blocked_by", "duplicates", "refines"} // validateStrict validates a frontmatter map against the issue schema. It // special-cases schema_version first (mirrors _validate_strict) and rejects @@ -142,6 +142,9 @@ func validateStrict(fm map[string]any) error { {"related_specs", reSpcID, "spc-N"}, {"related_issues", reIssID, "iss-N"}, {"blocked_by", reIssID, "iss-N"}, + // The filing-time match's typed links name an issue or an intent. + {"duplicates", reLinkID, "iss-N or itd-N"}, + {"refines", reLinkID, "iss-N or itd-N"}, } for _, f := range idListFields { v, present := fm[f.field] @@ -302,6 +305,8 @@ func issueFromFrontmatter(fm map[string]any, status State, path, body string) Is iss.RelatedSpecs = asStrList(fm["related_specs"]) iss.RelatedIssues = asStrList(fm["related_issues"]) iss.BlockedBy = asStrList(fm["blocked_by"]) + iss.Duplicates = asStrList(fm["duplicates"]) + iss.Refines = asStrList(fm["refines"]) if rb, ok := fm["resolved_by"].(map[string]any); ok { iss.ResolvedBy = &ResolvedBy{ Intent: asString(rb["intent"]), diff --git a/internal/core/capture/workflow.go b/internal/core/capture/workflow.go index 67b762f15..0d594a1f7 100644 --- a/internal/core/capture/workflow.go +++ b/internal/core/capture/workflow.go @@ -13,6 +13,7 @@ import ( "github.com/intentdriven/abcd/internal/core/grounds" "github.com/intentdriven/abcd/internal/core/issueschema" "github.com/intentdriven/abcd/internal/core/provenance" + "github.com/intentdriven/abcd/internal/core/record/match" "github.com/intentdriven/abcd/internal/core/relink" "github.com/intentdriven/abcd/internal/fsutil" "github.com/intentdriven/abcd/internal/termsafe" @@ -256,10 +257,17 @@ func commitCapture(repoRoot, issuesRoot string, req CaptureRequest, issID, slug, // Written inside the ledger's os.Root (iss-2609012037143368): an ancestor // swapped since the re-read above cannot carry the record out of the // checkout. + // The filing-time match (itd-2609212137116617) runs here, under the + // ledger lock, so the ledger it reads is the one the record joins. It + // adds links to the content and never fails the write. + var matched *match.Outcome + if req.Match != nil { + content, matched = matchAndLink(repoRoot, issuesRoot, *req.Match, req.Text, content, fm) + } if werr := writeLedgerFile(repoRoot, issuesRoot, placeholder, []byte(content)); werr != nil { return werr } - result = CaptureResult{ID: issID, Slug: slug, Path: placeholder, Status: StateOpen} + result = CaptureResult{ID: issID, Slug: slug, Path: placeholder, Status: StateOpen, Match: matched} return nil }) if err != nil { diff --git a/internal/core/intent/create.go b/internal/core/intent/create.go index e36cd14e7..660af41cc 100644 --- a/internal/core/intent/create.go +++ b/internal/core/intent/create.go @@ -11,6 +11,7 @@ import ( "github.com/intentdriven/abcd/internal/core/changelog" "github.com/intentdriven/abcd/internal/core/provenance" + "github.com/intentdriven/abcd/internal/core/record/match" "github.com/intentdriven/abcd/internal/core/recordid" "github.com/intentdriven/abcd/internal/termsafe" ) @@ -59,9 +60,16 @@ const maxSlugLen = 60 // `abcd intent plan` schedules it. This is the quoted-text create path itd-46 // delivers — the create half of what spc-6 AC3 (promote) needs. func CreateFromText(repoRoot, text string, opts TextOptions) (Intent, error) { + c, err := createFromText(repoRoot, text, opts, nil) + return c.Intent, err +} + +// createFromText is CreateFromText with the optional filing-time match +// (match.go), which runs inside CreateDraft's mint lock. +func createFromText(repoRoot, text string, opts TextOptions, m *Matcher) (Created, error) { trimmed := strings.TrimSpace(text) if trimmed == "" { - return Intent{}, fmt.Errorf("intent: refusing to create from empty text") + return Created{}, fmt.Errorf("intent: refusing to create from empty text") } // An explicit title is held to the text's own bar before anything is // derived: non-empty once trimmed, and one line — it becomes the H1, where a @@ -71,10 +79,10 @@ func CreateFromText(repoRoot, text string, opts TextOptions) (Intent, error) { // alike rather than one of them silently falling back to the derived H1. title := strings.TrimSpace(opts.Title) if (opts.TitleSet || opts.Title != "") && title == "" { - return Intent{}, fmt.Errorf("intent: refusing an empty --title (nothing written)") + return Created{}, fmt.Errorf("intent: refusing an empty --title (nothing written)") } if strings.ContainsAny(title, "\r\n") { - return Intent{}, fmt.Errorf("intent: --title must be a single line (nothing written)") + return Created{}, fmt.Errorf("intent: --title must be a single line (nothing written)") } // Redact the caller's text through the one canonical scanner BEFORE anything // derived from it is built (gh-486). The slug becomes the filename and is @@ -85,11 +93,11 @@ func CreateFromText(repoRoot, text string, opts TextOptions) (Intent, error) { // body again as its own boundary guard; that second pass is idempotent here. redacted, _, err := redactIntentText(repoRoot, trimmed) if err != nil { - return Intent{}, err + return Created{}, err } slug, err := deriveIntentSlug(redacted) if err != nil { - return Intent{}, err + return Created{}, err } if title == "" { title = deriveTitle(redacted) @@ -97,10 +105,10 @@ func CreateFromText(repoRoot, text string, opts TextOptions) (Intent, error) { // The same boundary the text crossed: the title is free prose too, and // CreateDraft's own pass is idempotent on it. if title, _, err = redactIntentText(repoRoot, title); err != nil { - return Intent{}, err + return Created{}, err } } - return CreateDraft(repoRoot, DraftOptions{ + it, outcome, err := createDraftMatched(repoRoot, DraftOptions{ Slug: slug, Title: title, PressRelease: redacted, @@ -109,7 +117,9 @@ func CreateFromText(repoRoot, text string, opts TextOptions) (Intent, error) { // the default; the production mode is the operator's declared one, or the // repo's default resolved by the surface. ProductionMode: opts.ProductionMode, + Match: m, }) + return Created{Intent: it, Match: outcome}, err } // TextOptions parameterizes CreateFromText beyond the text itself: an explicit @@ -193,6 +203,11 @@ type DraftOptions struct { // provenance.DefaultMode, so a draft written through a command carries the key // whatever the caller says. ProductionMode string + // Match, when non-nil, matches the draft's title and press release against + // the record under the mint lock and writes each likely double as a typed + // link (match.go). The promote route passes none: its draft is joined to + // the record it graduated from already. + Match *Matcher } // relatedIssueRe constrains the promote back-edge to a captured id before it is @@ -211,17 +226,24 @@ var relatedIssueRe = regexp.MustCompile(`^(iss|rdi)-[0-9]+$`) // id through the shared recordid seam, and atomically writes // drafts/itd-N-.md. On any refusal nothing is written. func CreateDraft(repoRoot string, opts DraftOptions) (Intent, error) { + it, _, err := createDraftMatched(repoRoot, opts) + return it, err +} + +// createDraftMatched is CreateDraft returning the filing-time match's outcome too, +// nil when opts asked for none. +func createDraftMatched(repoRoot string, opts DraftOptions) (Intent, *match.Outcome, error) { if !slugRe.MatchString(opts.Slug) { - return Intent{}, fmt.Errorf("intent: slug %q is not kebab-case", opts.Slug) + return Intent{}, nil, fmt.Errorf("intent: slug %q is not kebab-case", opts.Slug) } if strings.TrimSpace(opts.Title) == "" { - return Intent{}, fmt.Errorf("intent: refusing to create a draft with an empty title") + return Intent{}, nil, fmt.Errorf("intent: refusing to create a draft with an empty title") } if strings.TrimSpace(opts.SeedBody) == "" && strings.TrimSpace(opts.PressRelease) == "" { - return Intent{}, fmt.Errorf("intent: refusing to create a draft with neither a press release nor a seed body") + return Intent{}, nil, fmt.Errorf("intent: refusing to create a draft with neither a press release nor a seed body") } if opts.RelatedIssue != "" && !relatedIssueRe.MatchString(opts.RelatedIssue) { - return Intent{}, fmt.Errorf("intent: related issue %q must match ^(iss|rdi)-[0-9]+$", opts.RelatedIssue) + return Intent{}, nil, fmt.Errorf("intent: related issue %q must match ^(iss|rdi)-[0-9]+$", opts.RelatedIssue) } // impact is optional on a draft (intent_impact_valid gates the move into // shipped/, not the seed), but when set it must be a legal, non-internal @@ -231,10 +253,10 @@ func CreateDraft(repoRoot string, opts DraftOptions) (Intent, error) { if opts.Impact != "" { imp, err := changelog.ParseImpact(opts.Impact) if err != nil { - return Intent{}, fmt.Errorf("intent: %w", err) + return Intent{}, nil, fmt.Errorf("intent: %w", err) } if imp == changelog.ImpactInternal { - return Intent{}, fmt.Errorf("intent: impact must not be internal on an intent — a press-release-first intent is user-facing by definition; declare one of additive|breaking|fix, or record the work as an issue instead") + return Intent{}, nil, fmt.Errorf("intent: impact must not be internal on an intent — a press-release-first intent is user-facing by definition; declare one of additive|breaking|fix, or record the work as an issue instead") } } @@ -244,7 +266,7 @@ func CreateDraft(repoRoot string, opts DraftOptions) (Intent, error) { // path; an unset mode means provenance.DefaultMode. stamp, err := draftStamp(opts) if err != nil { - return Intent{}, fmt.Errorf("intent: %w", err) + return Intent{}, nil, fmt.Errorf("intent: %w", err) } // Boundary redaction (gh-486): CreateDraft is the ONE canonical draft-mint @@ -257,15 +279,15 @@ func CreateDraft(repoRoot string, opts DraftOptions) (Intent, error) { // scanner; the pass is idempotent for text a caller already redacted. rTitle, _, err := redactIntentText(repoRoot, opts.Title) if err != nil { - return Intent{}, err + return Intent{}, nil, err } rPress, _, err := redactIntentText(repoRoot, opts.PressRelease) if err != nil { - return Intent{}, err + return Intent{}, nil, err } rBody, _, err := redactIntentText(repoRoot, opts.SeedBody) if err != nil { - return Intent{}, err + return Intent{}, nil, err } // Hidden runes — a bidi override, a zero-width rune, a C1 control, DEL — are // percent-encoded at the same boundary, after redaction, with termsafe's one @@ -276,6 +298,7 @@ func CreateDraft(repoRoot string, opts DraftOptions) (Intent, error) { opts.SeedBody = termsafe.EncodeHiddenRunesBlock(rBody) var created Intent + var outcome *match.Outcome err = withIntentMintLock(repoRoot, func() error { draftsDirAbs := filepath.Join(repoRoot, IntentsRelDir, BucketDrafts) if err := ensureRecordDir(repoRoot, filepath.Join(IntentsRelDir, BucketDrafts)); err != nil { @@ -292,7 +315,14 @@ func CreateDraft(repoRoot string, opts DraftOptions) (Intent, error) { name := id + "-" + opts.Slug + ".md" rel := filepath.Join(IntentsRelDir, BucketDrafts, name) abs := filepath.Join(draftsDirAbs, name) - content := seedDraft(id, opts, stamp) + // The filing-time match runs here, under the mint lock, so the record + // it reads is the record the draft is written into. + var links map[match.Relation][]string + if opts.Match != nil { + outcome = runMatch(opts.Match, opts.Title+"\n"+opts.PressRelease) + links = outcome.Links() + } + content := seedDraft(id, opts, stamp, links) if err := writeIntentFile(abs, rel, content); err != nil { return err } @@ -310,9 +340,9 @@ func CreateDraft(repoRoot string, opts DraftOptions) (Intent, error) { return nil }) if err != nil { - return Intent{}, err + return Intent{}, nil, err } - return created, Validate(created) + return created, outcome, Validate(created) } // draftStamp resolves the draft's disclosure pair from the mint options: the @@ -413,7 +443,7 @@ var intentFileNumRe = recordid.FilenameNumRe(intentFamily) // Press Release prose or the route's seed note, the Why This Matters seed body // or its prompt, and the itd-1 discipline's Acceptance Criteria section left as // a placeholder for the human to fill before planning. -func seedDraft(id string, opts DraftOptions, stamp provenance.Stamp) string { +func seedDraft(id string, opts DraftOptions, stamp provenance.Stamp, links map[match.Relation][]string) string { var b strings.Builder b.WriteString("---\n") b.WriteString("id: " + id + "\n") @@ -431,6 +461,13 @@ func seedDraft(id string, opts DraftOptions, stamp provenance.Stamp) string { if opts.RelatedIssue != "" { b.WriteString(RelatedIssuesKey + ": [" + opts.RelatedIssue + "]\n") } + // The filing-time match's typed links (itd-2609212137116617), an inline id + // list each, written only when a candidate cleared the threshold. + for _, rel := range []match.Relation{match.Duplicates, match.Refines} { + if ids := links[rel]; len(ids) > 0 { + b.WriteString(string(rel) + ": [" + strings.Join(ids, ", ") + "]\n") + } + } // impact is written only when the caller declared one (validated in // CreateDraft). It is bare — the machine-read enum the shipped-intent gate // compares byte-for-byte — and travels unchanged to shipped/. An unset impact diff --git a/internal/core/intent/match.go b/internal/core/intent/match.go new file mode 100644 index 000000000..2d50ca85e --- /dev/null +++ b/internal/core/intent/match.go @@ -0,0 +1,126 @@ +package intent + +import ( + "path/filepath" + "regexp" + "strings" + + "github.com/intentdriven/abcd/internal/core/frontmatter" + "github.com/intentdriven/abcd/internal/core/mdrecord" + "github.com/intentdriven/abcd/internal/core/record/match" +) + +// match.go is the intent store's half of the filing-time match +// (itd-2609212137116617): the text every intent offers as a candidate, and the +// quoted-text create that matches a new draft against the record before it is +// written. The score is internal/core/record/match's; the ledger gathers the +// candidate set, because the ledger reads intents and this store does not read +// the ledger. + +// MatchText is one intent's comparable text: its H1 and its press release. +// A press release that is still the seed note a create wrote is left out, since +// every promoted draft carries the same words there. +type MatchText struct { + ID string + Title string + PressRelease string +} + +var ( + h1Re = regexp.MustCompile(`^#\s+(.+?)\s*$`) + pressReleaseRe = regexp.MustCompile(`^##\s+Press Release\s*$`) + seedReduceRe = regexp.MustCompile(`[>_*\s]+`) +) + +// MatchTexts reads the title and press release of every intent in every +// bucket. A store no reader can load is an error, which the match reports as +// its reason for comparing nothing, never as a refusal of the write. +func MatchTexts(repoRoot string) ([]MatchText, error) { + c, err := Load(repoRoot) + if err != nil { + return nil, err + } + out := make([]MatchText, 0, len(c.Intents)) + for _, it := range c.Intents { + data, err := readRepoFile(filepath.Join(repoRoot, it.Path), it.Path) + if err != nil { + return nil, err + } + out = append(out, matchTextOf(it.ID, string(data))) + } + return out, nil +} + +func matchTextOf(id, content string) MatchText { + lines := strings.Split(content, "\n") + // The body starts after the frontmatter's closing delimiter. + start := 0 + if len(lines) > 0 && frontmatter.IsDelimiter(frontmatter.TrimBOM(lines[0])) { + for i := 1; i < len(lines); i++ { + if frontmatter.IsDelimiter(lines[i]) { + start = i + 1 + break + } + } + } + body := lines[start:] + mt := MatchText{ID: id} + mask := mdrecord.Mask(body) + for i, ln := range body { + if mask[i] != 0 { + continue + } + if m := h1Re.FindStringSubmatch(strings.TrimRight(ln, "\r")); m != nil { + mt.Title = m[1] + break + } + } + if s, e, ok := mdrecord.SectionLineRangeIn(body, mask, pressReleaseRe); ok { + press := strings.TrimSpace(strings.Join(body[s:e], "\n")) + if !IsSeedNote(strings.TrimSpace(seedReduceRe.ReplaceAllString(press, " "))) { + mt.PressRelease = press + } + } + return mt +} + +// Matcher is a create's request to be matched before it is written: the +// threshold, and the candidate set, which the caller's reader gathers and the +// create calls under the store's mint lock. +type Matcher struct { + Threshold float64 + Candidates func() ([]match.Candidate, error) +} + +// Created is a quoted-text create's result: the draft, and the match's +// outcome when one was asked for. +type Created struct { + Intent + Match *match.Outcome `json:"match,omitempty"` +} + +// CreateFromTextMatched is CreateFromText with the filing-time match: the new +// draft's title and press release are compared with the candidates under the +// mint lock, and each likely double is written onto the draft as a +// `duplicates:` or `refines:` link. The match never refuses the create: a +// candidate set that cannot be read is reported on the outcome and the draft +// is written unlinked. A nil matcher is CreateFromText exactly. +func CreateFromTextMatched(repoRoot, text string, opts TextOptions, m *Matcher) (Created, error) { + return createFromText(repoRoot, text, opts, m) +} + +// runMatch runs m over text, turning a candidate read that fails into an +// outcome that says so rather than an error. +func runMatch(m *Matcher, text string) *match.Outcome { + if match.Short(text) { + o := match.Rank(text, nil, m.Threshold) + return &o + } + cands, err := m.Candidates() + if err != nil { + o := match.Unread(m.Threshold, err) + return &o + } + o := match.Rank(text, cands, m.Threshold) + return &o +} diff --git a/internal/core/intent/match_test.go b/internal/core/intent/match_test.go new file mode 100644 index 000000000..390242399 --- /dev/null +++ b/internal/core/intent/match_test.go @@ -0,0 +1,139 @@ +package intent + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/core/lint" + "github.com/intentdriven/abcd/internal/core/record/match" +) + +const ( + existingTitle = "The quiet board says who holds the next move" + existingPress = "The bare board names the record whose next move is waiting on a person, " + + "says which person, and prints the command that answers it, without opening a file." + newDraftText = "The bare board names the record whose next move waits on a person, " + + "says which person that is, and prints the command that answers it, without opening any file." +) + +func fixedCandidates(cands ...match.Candidate) func() ([]match.Candidate, error) { + return func() ([]match.Candidate, error) { + return append([]match.Candidate{ + {ID: "itd-21", Text: "The site builder renders a stale anchor for a heading renamed since the last build."}, + {ID: "iss-22", Text: "The history store drops a transcript that exceeds its byte budget without saying so."}, + }, cands...), nil + } +} + +// Criterion 2: a draft whose text overlaps an existing intent's title and +// press release is written with the link, and the result carries the match. +func TestCreateFromTextMatchedLinksADouble(t *testing.T) { + root := t.TempDir() + m := &Matcher{Threshold: match.DefaultThreshold, Candidates: fixedCandidates( + match.Candidate{ID: "itd-20", Text: existingTitle + "\n" + existingPress})} + c, err := CreateFromTextMatched(root, newDraftText, TextOptions{}, m) + if err != nil { + t.Fatalf("create refused: %v", err) + } + if c.Match == nil || len(c.Match.Matches) != 1 || c.Match.Matches[0].ID != "itd-20" || + c.Match.Matches[0].Relation != match.Duplicates { + t.Fatalf("match = %+v, want itd-20 as duplicates", c.Match) + } + data, err := os.ReadFile(filepath.Join(root, c.Path)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "\nduplicates: [itd-20]\n") { + t.Fatalf("the draft carries no link:\n%s", data) + } + // The linked draft is as lint-valid as an unlinked one. + cfg := lint.Config{ + Roots: []string{".abcd/development"}, + Rules: map[string]lint.RuleConfig{ + "intent_lifecycle": {Enabled: true, Severity: "blocker", IntentsDir: "intents"}, + }, + } + findings, err := lint.Lint(cfg, root) + if err != nil { + t.Fatal(err) + } + for _, f := range findings { + t.Fatalf("linked draft: %s:%d %s", f.File, f.Line, f.Message) + } +} + +// Criteria 3 and 4: a candidate set that cannot be read never refuses the +// create, and below the threshold nothing is written while the near misses +// are listed. +func TestCreateFromTextMatchedNeverRefuses(t *testing.T) { + root := t.TempDir() + broken := &Matcher{Threshold: match.DefaultThreshold, Candidates: func() ([]match.Candidate, error) { + return nil, errors.New("the ledger is unreadable") + }} + c, err := CreateFromTextMatched(root, newDraftText, TextOptions{}, broken) + if err != nil { + t.Fatalf("an unreadable candidate set refused the create: %v", err) + } + if c.Match == nil || !strings.Contains(c.Match.Skipped, "unreadable") { + t.Fatalf("outcome = %+v, want the reason nothing was compared", c.Match) + } + + strict := &Matcher{Threshold: 0.99, Candidates: fixedCandidates( + match.Candidate{ID: "itd-20", Text: existingTitle + "\n" + existingPress})} + c, err = CreateFromTextMatched(root, newDraftText+" Also a second time.", TextOptions{}, strict) + if err != nil { + t.Fatal(err) + } + if len(c.Match.Matches) != 0 || len(c.Match.NearMisses) == 0 || c.Match.NearMisses[0].ID != "itd-20" { + t.Fatalf("outcome = %+v, want itd-20 as a near miss only", c.Match) + } + data, _ := os.ReadFile(filepath.Join(root, c.Path)) + if strings.Contains(string(data), "duplicates:") || strings.Contains(string(data), "refines:") { + t.Fatalf("a link was written below the threshold:\n%s", data) + } +} + +// Without a matcher the create is CreateFromText exactly. +func TestCreateFromTextMatchedWithoutAMatcher(t *testing.T) { + c, err := CreateFromTextMatched(t.TempDir(), newDraftText, TextOptions{}, nil) + if err != nil || c.Match != nil { + t.Fatalf("err %v, match %+v: want a plain create", err, c.Match) + } +} + +// MatchTexts offers each intent's H1 and press release, and leaves out a press +// release that is still a create's seed note. +func TestMatchTextsReadsTitleAndPressRelease(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, IntentsRelDir, BucketPlanned) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + write := func(name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("itd-5-a.md", "---\nid: itd-5\nslug: a\n---\n\n# "+existingTitle+"\n\n## Press Release\n\n> "+ + existingPress+"\n\n## Why This Matters\n\nNot compared.\n") + write("itd-6-b.md", "---\nid: itd-6\nslug: b\n---\n\n# Promoted\n\n## Press Release\n\n> _"+ + promotionSeedOpening+"iss-9. "+seedNoteTail+"_\n") + texts, err := MatchTexts(root) + if err != nil { + t.Fatal(err) + } + got := map[string]MatchText{} + for _, mt := range texts { + got[mt.ID] = mt + } + if got["itd-5"].Title != existingTitle || !strings.Contains(got["itd-5"].PressRelease, "prints the command") || + strings.Contains(got["itd-5"].PressRelease, "Not compared") { + t.Fatalf("itd-5 = %+v", got["itd-5"]) + } + if got["itd-6"].Title != "Promoted" || got["itd-6"].PressRelease != "" { + t.Fatalf("itd-6 = %+v, want its seed note left out", got["itd-6"]) + } +} diff --git a/internal/core/intent/ready_test.go b/internal/core/intent/ready_test.go index 5e92eed2e..37ea4f1d0 100644 --- a/internal/core/intent/ready_test.go +++ b/internal/core/intent/ready_test.go @@ -518,7 +518,7 @@ func TestReadyClaimChecksNotApplicableInTerminalBuckets(t *testing.T) { // and the gate must not report it as one. func TestReadyScaffoldPromptIsNotAClaim(t *testing.T) { root := t.TempDir() - seeded := seedDraft("itd-10", DraftOptions{Slug: "alpha", Title: "alpha", SeedBody: "why it matters"}, researcherStamp(t)) + seeded := seedDraft("itd-10", DraftOptions{Slug: "alpha", Title: "alpha", SeedBody: "why it matters"}, researcherStamp(t), nil) // Plan the criteria only, so the two claim sections stay as seeded. seeded = strings.Replace(seeded, "> _Required (the itd-1 discipline)", "- **Given** x, **when** y, **then** z.\n\n> _was: ", 1) diff --git a/internal/core/issueschema/issueschema.go b/internal/core/issueschema/issueschema.go index 56afad29b..b6ee2f1b1 100644 --- a/internal/core/issueschema/issueschema.go +++ b/internal/core/issueschema/issueschema.go @@ -69,6 +69,12 @@ var Known = map[string]bool{ "related_specs": true, "related_issues": true, "synthesis_clusters": true, "wontfix_reason": true, "resolution": true, "resolved_by": true, "blocked_by": true, + // duplicates and refines are the typed links the filing-time match writes + // (itd-2609212137116617): each an id list naming the issue or intent this + // record is a near-identical double of, or a narrower case of. Optional, and + // removable by hand — a record without either is ordinary — but KNOWN, or + // the reader would drop every linked record as malformed. + "duplicates": true, "refines": true, // shipped_in names the release that already carried this record's work, so the // derivation can leave it out of a later cut (iss-2608241612087533). Optional // and rare — only a ledger-hygiene close, for a fix released long ago, has diff --git a/internal/core/lint/matchlinks_test.go b/internal/core/lint/matchlinks_test.go new file mode 100644 index 000000000..4ef6eba60 --- /dev/null +++ b/internal/core/lint/matchlinks_test.go @@ -0,0 +1,27 @@ +package lint + +import ( + "path/filepath" + "testing" +) + +// The filing-time match's typed links (itd-2609212137116617) are claims that +// another record exists, like every other cross-reference: a `duplicates:` or +// `refines:` naming a record the corpus does not hold is a finding, and one +// naming a present issue or intent is silent. +func TestRecordSchemaResolvesMatchLinks(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "rec/intents/drafts/itd-17-present.md", "---\nid: itd-17\nkind: null\nspec_id: null\n---\n# present\n") + writeFile(t, root, "rec/intents/drafts/itd-18-linked.md", + "---\nid: itd-18\nkind: null\nspec_id: null\nduplicates: [itd-17]\nrefines: [itd-99]\n---\n# linked\n") + fs, err := Lint(schemaConfig(), root) + if err != nil { + t.Fatal(err) + } + if n := countRule(fs, ruleRecordSchema); n != 1 { + t.Fatalf("want exactly the one phantom refines target, got %d: %+v", n, fs) + } + if !findingWith(fs, filepath.Join("rec/intents/drafts", "itd-18-linked.md"), ruleRecordSchema, "refines names 'itd-99'") { + t.Fatalf("no finding naming the phantom refines target: %+v", fs) + } +} diff --git a/internal/core/lint/schema.go b/internal/core/lint/schema.go index 85107fb83..e367054e3 100644 --- a/internal/core/lint/schema.go +++ b/internal/core/lint/schema.go @@ -113,7 +113,9 @@ var ( // pruned id, so it can never fail its own resolution check; listing it would // read as coverage it does not provide. Its targets are checked separately, // against the store's allocation high-water mark. - recordRefFields = []string{"related_adrs", "related_intents", "builds_on", "blocked_by"} + // `duplicates` and `refines` are the typed links the filing-time match + // writes (itd-2609212137116617), each naming an issue or an intent. + recordRefFields = []string{"related_adrs", "related_intents", "builds_on", "blocked_by", "duplicates", "refines"} // Every field the rule reads handles out of, so the scan parses each once. recordHandleFields = append([]string{"supersedes", "superseded_by"}, recordRefFields...) // recordGraphFields are cross-reference fields the record carries that this diff --git a/internal/core/record/match/match.go b/internal/core/record/match/match.go index ab105e260..2ca19d76c 100644 --- a/internal/core/record/match/match.go +++ b/internal/core/record/match/match.go @@ -244,6 +244,11 @@ func Overlap(a, b []string, w Weights) (forward, reverse float64, shared []strin return forward, reverse, shared } +// Short reports whether text carries fewer distinct terms than MinTerms, so a +// caller can file it without gathering any candidates: Rank would compare +// nothing anyway. +func Short(text string) bool { return len(Terms(text)) < MinTerms } + // Rank matches text against every candidate. A threshold outside (0, 1] is // the caller's fault and reads as the bundled default, so a matcher can never // be configured into linking everything or nothing by accident; the layered @@ -314,3 +319,17 @@ func Rank(text string, cands []Candidate, threshold float64) Outcome { } func round3(f float64) float64 { return math.Round(f*1000) / 1000 } + +// Unread is the outcome of a match whose candidate set could not be read: the +// record is filed without matching, and the reason travels with the outcome +// rather than refusing the write. +func Unread(threshold float64, err error) Outcome { + if !(threshold > 0 && threshold <= 1) { + threshold = DefaultThreshold + } + return Outcome{ + Heuristic: Heuristic, Threshold: threshold, MinTerms: MinTerms, + Skipped: "the record could not be read for matching (" + err.Error() + "), so it is filed without matching", + Matches: []Score{}, NearMisses: []Score{}, + } +} From bea449bf3890c798f6726647849ec5066b6ad01e Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:03:50 +0100 Subject: [PATCH 11/60] chore: close spc-2609211955339422 and ship itd-63 The explain-then-install mode, its registry and its callers are on the branch. The close note records the two places the tree differs from the record the spec was written from: the itd-62 safety gate is a draft, so the history store's armed-gitleaks refusal is the missing-scanner path routed; and the guard and the launch have no tool checks to reroute. Delivers: itd-63 Assisted-by: Claude:claude-opus-5-5 --- .../itd-63-setup-wizard-explains-installs.md | 3 ++- .../plans/2026-08-11-install-experience.md | 2 +- .../2026-08-15-facilitator-experience.md | 2 +- ...55339422-setup-wizard-explains-installs.md | 24 +++++++++++++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) rename .abcd/development/intents/{planned => shipped}/itd-63-setup-wizard-explains-installs.md (98%) rename .abcd/development/specs/{open => closed}/spc-2609211955339422-setup-wizard-explains-installs.md (60%) diff --git a/.abcd/development/intents/planned/itd-63-setup-wizard-explains-installs.md b/.abcd/development/intents/shipped/itd-63-setup-wizard-explains-installs.md similarity index 98% rename from .abcd/development/intents/planned/itd-63-setup-wizard-explains-installs.md rename to .abcd/development/intents/shipped/itd-63-setup-wizard-explains-installs.md index 69e8fff29..e92775125 100644 --- a/.abcd/development/intents/planned/itd-63-setup-wizard-explains-installs.md +++ b/.abcd/development/intents/shipped/itd-63-setup-wizard-explains-installs.md @@ -76,7 +76,8 @@ _None open; decisions 1 to 3 settle the three this record carried._ ## Audit Notes -_Empty. Populated by intent-fidelity-reviewer when intent moves to shipped/._ + +Fidelity review OWED (receipt rcp-3c9fb4ba9770). ### Linkage note (spc-83.5) diff --git a/.abcd/development/plans/2026-08-11-install-experience.md b/.abcd/development/plans/2026-08-11-install-experience.md index 746c73a80..2d76c117f 100644 --- a/.abcd/development/plans/2026-08-11-install-experience.md +++ b/.abcd/development/plans/2026-08-11-install-experience.md @@ -70,7 +70,7 @@ assume, and the contradictions are the point. mechanism the same release removes. 8. **iss-163/iss-164 (plain-language prompt help, persona-readable summary) are out of both cuts**, deferred to - [itd-63](../intents/planned/itd-63-setup-wizard-explains-installs.md). + [itd-63](../intents/shipped/itd-63-setup-wizard-explains-installs.md). Real, and larger than everything else here combined; keeping them out keeps both cuts reviewable. diff --git a/.abcd/development/plans/2026-08-15-facilitator-experience.md b/.abcd/development/plans/2026-08-15-facilitator-experience.md index 422760c7f..5772ba1a9 100644 --- a/.abcd/development/plans/2026-08-15-facilitator-experience.md +++ b/.abcd/development/plans/2026-08-15-facilitator-experience.md @@ -44,7 +44,7 @@ Ordering and item specs unchanged from that plan: — canonical per-choice help text lives in core; the foundation item. 2. **[iss-164](../../work/issues/open/iss-164-the-ahoy-install-completion-summary-is-written-for-abcd-s-im.md)** (blocked by iss-163) — persona-readable result summaries. -3. **[itd-63](../intents/planned/itd-63-setup-wizard-explains-installs.md)** +3. **[itd-63](../intents/shipped/itd-63-setup-wizard-explains-installs.md)** — the intent frame A1/A2 deliver into. Lifecycle first: planned but spec-less, so the first milestone is the spec and `intent ready`, never code. diff --git a/.abcd/development/specs/open/spc-2609211955339422-setup-wizard-explains-installs.md b/.abcd/development/specs/closed/spc-2609211955339422-setup-wizard-explains-installs.md similarity index 60% rename from .abcd/development/specs/open/spc-2609211955339422-setup-wizard-explains-installs.md rename to .abcd/development/specs/closed/spc-2609211955339422-setup-wizard-explains-installs.md index 0b8969b78..1e03b0ea1 100644 --- a/.abcd/development/specs/open/spc-2609211955339422-setup-wizard-explains-installs.md +++ b/.abcd/development/specs/closed/spc-2609211955339422-setup-wizard-explains-installs.md @@ -52,3 +52,27 @@ each with a test that the explanation appears and the default holds on a no. | 3 unknown tool: generic text, gap captured | scope 1 | | 4 the safety gate routes through it | scope 3 | | 5 no host dependency | scope 2 | + +## Close note + +Delivered on 2026-09-26 against the tree as it stands, which differs from the +record this spec was written from in two places: + +- **The safety gate's missing-scanner path** (criterion 4) belongs to itd-62, + which is still a draft: no gate on the default branch always blocks on a + missing scanner. The one fail-closed missing-scanner path there is the + history store's, for a repository that armed the gitleaks adapter + (`internal/adapter/gitleaks`, opt-in since the 2026-07-24 ruling that made + the native scanner the default). Its refusal now carries the registry's + explanation (`tools.Missing`), and `ahoy install` offers the install, as a + required tool, for exactly that repository. +- **The guard's and the launch's tool checks** (scope 3) do not exist: the + guard runs no external tool, and the launch scans are native (itd-65). There + was nothing to reroute, and nothing was invented. + +The other callers are rerouted: `ahoy`'s dependency gap and install step, and +the missing-`gh` refusal of `ahoy remote` and `site setup`. The trufflehog gap +was removed rather than routed (iss-2609261447331434): nothing runs trufflehog. +The unknown-tool gap (criterion 3) is captured, not composed: the explanation +names it as abcd's own and carries the `abcd capture` line that records it, +and a test fails on any tool ahoy names that the registry lacks. From fc9ceb16be0ea650ef91012c3eca8ea7a6f41acb Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:04:01 +0100 Subject: [PATCH 12/60] style: sort the history store's imports Assisted-by: Claude:claude-opus-5-5 --- internal/core/history/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/core/history/history.go b/internal/core/history/history.go index 5e3880259..99a8e5285 100644 --- a/internal/core/history/history.go +++ b/internal/core/history/history.go @@ -33,8 +33,8 @@ import ( "time" "github.com/intentdriven/abcd/internal/adapter/gitleaks" - "github.com/intentdriven/abcd/internal/core/tools" "github.com/intentdriven/abcd/internal/adapter/scanner" + "github.com/intentdriven/abcd/internal/core/tools" "github.com/intentdriven/abcd/internal/fsutil" ) From 21db95647a60b66aeb83cc5a808ac9c6ee27eae6 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:11:30 +0100 Subject: [PATCH 13/60] feat(cli): capture and the intent create print the filing-time match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both verbs resolve match.threshold and match.fields through the layered reader and hand them to core, which runs the match under the writer's lock. The text render prints each link written ("matched iss-N — duplicates (score …): link written; confirm it by leaving the link, or remove the line"), a match past the link cap as listed-not-linked, the count of near misses, or why nothing was compared; --json carries the whole outcome, near misses and their scores included. A configuration the reader refuses (a misspelt key, a threshold out of range) never refuses the filing: the record is filed unlinked and the outcome names the refused key, because a finding lost to a config typo is the failure the match exists to prevent. No new flag. Assisted-by: Claude:claude-opus-5-5 --- internal/core/record/match/match.go | 9 +- internal/surface/cli/cli.go | 24 +++- internal/surface/cli/match.go | 58 ++++++++++ internal/surface/cli/match_cli_test.go | 145 +++++++++++++++++++++++++ 4 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 internal/surface/cli/match.go create mode 100644 internal/surface/cli/match_cli_test.go diff --git a/internal/core/record/match/match.go b/internal/core/record/match/match.go index 2ca19d76c..4b57d40f0 100644 --- a/internal/core/record/match/match.go +++ b/internal/core/record/match/match.go @@ -324,12 +324,17 @@ func round3(f float64) float64 { return math.Round(f*1000) / 1000 } // record is filed without matching, and the reason travels with the outcome // rather than refusing the write. func Unread(threshold float64, err error) Outcome { + return Skip(threshold, "the record could not be read for matching ("+err.Error()+"), so it is filed without matching") +} + +// Skip is the outcome of a match that compared nothing, for the reason given: +// the filing goes ahead, and the reason is what a surface says. +func Skip(threshold float64, reason string) Outcome { if !(threshold > 0 && threshold <= 1) { threshold = DefaultThreshold } return Outcome{ Heuristic: Heuristic, Threshold: threshold, MinTerms: MinTerms, - Skipped: "the record could not be read for matching (" + err.Error() + "), so it is filed without matching", - Matches: []Score{}, NearMisses: []Score{}, + Skipped: reason, Matches: []Score{}, NearMisses: []Score{}, } } diff --git a/internal/surface/cli/cli.go b/internal/surface/cli/cli.go index 3aee32488..703ee3da9 100644 --- a/internal/surface/cli/cli.go +++ b/internal/surface/cli/cli.go @@ -38,6 +38,7 @@ import ( "github.com/intentdriven/abcd/internal/core/oracle" "github.com/intentdriven/abcd/internal/core/provenance" "github.com/intentdriven/abcd/internal/core/record" + "github.com/intentdriven/abcd/internal/core/record/match" "github.com/intentdriven/abcd/internal/core/rules" "github.com/intentdriven/abcd/internal/core/spec" "github.com/intentdriven/abcd/internal/core/surface" @@ -2501,12 +2502,25 @@ func createIntentFromText(cmd *cobra.Command, repoRoot, text string, opts intent return err } opts.ProductionMode = mode - it, err := intent.CreateFromText(repoRoot, text, opts) + // The filing-time match (itd-2609212137116617): the ledger gathers the + // candidates, the create runs the match under its mint lock. + var m *intent.Matcher + cfg, refused := resolveMatch(cmd.ErrOrStderr(), "intent", repoRoot) + if cfg != nil { + m = &intent.Matcher{Threshold: cfg.Threshold, Candidates: func() ([]match.Candidate, error) { + return capture.MatchCandidates(repoRoot, *cfg) + }} + } + it, err := intent.CreateFromTextMatched(repoRoot, text, opts, m) if err != nil { return &exitError{Code: 2, Msg: "abcd intent: " + err.Error()} } + if it.Match == nil { + it.Match = refused + } return render(cmd.OutOrStdout(), asJSON, it, func(w io.Writer) { fmt.Fprintf(w, "created %s (%s) — %s\n", it.ID, it.Bucket, termsafe.Sanitize(it.Path)) + renderMatch(w, it.Match) }) } @@ -3727,6 +3741,10 @@ func newCaptureCommand(asJSON *bool) *cobra.Command { if req.ProductionMode, err = resolveProductionMode(repoRoot, captureProductionMode); err != nil { return err } + // The filing-time match (itd-2609212137116617): configured through the + // layered reader, run by core under the ledger lock, never a refusal. + var matchRefused *match.Outcome + req.Match, matchRefused = resolveMatch(cmd.ErrOrStderr(), "capture", repoRoot) // --lapsed-at has NO default and is never filled in for the caller: a // lapse capture that omits the instant records none. The refusal that // stood here is parked, not lifted (iss-2609091009111294): the instant stays @@ -3743,8 +3761,12 @@ func newCaptureCommand(asJSON *bool) *cobra.Command { } return err } + if res.Match == nil { + res.Match = matchRefused + } return renderLedger(cmd.OutOrStdout(), *asJSON, repoRoot, res, func(w io.Writer) { fmt.Fprintf(w, "captured %s (%s) — %s\n", res.ID, res.Status, termsafe.Sanitize(res.Path)) + renderMatch(w, res.Match) // Folder membership is a status only once the file is committed // (iss-2609100508570527): say so at the write, where it is cheap. if res.Uncommitted { diff --git a/internal/surface/cli/match.go b/internal/surface/cli/match.go new file mode 100644 index 000000000..fddf5731d --- /dev/null +++ b/internal/surface/cli/match.go @@ -0,0 +1,58 @@ +package cli + +import ( + "fmt" + "io" + + "github.com/intentdriven/abcd/internal/core/layered" + "github.com/intentdriven/abcd/internal/core/record/match" + "github.com/intentdriven/abcd/internal/fsutil" + "github.com/intentdriven/abcd/internal/termsafe" +) + +// resolveMatch reads match.threshold and match.fields for a filing through the +// layered configuration reader (itd-2609212137116617). A configuration the +// reader refuses never refuses the filing, because a finding lost to a +// misspelt key is the failure the match exists to prevent: it comes back as +// the reason the match compared nothing, which the verb prints, and the record +// is filed unlinked. +func resolveMatch(stderr io.Writer, verb, repoRoot string) (*match.Config, *match.Outcome) { + roots, notes := layered.RootsFor(repoRoot) + for _, n := range notes { + fmt.Fprintf(stderr, "abcd %s\n", termsafe.Sanitize(fsutil.RedactHome(n))) + } + cfg, err := match.LoadConfig(roots) + if err != nil { + o := match.Skip(match.DefaultThreshold, "the match configuration is refused ("+ + fsutil.RedactHome(err.Error())+"), so the record is filed without matching; fix or remove the key") + return nil, &o + } + return &cfg, nil +} + +// renderMatch prints a filing's match: each link written, each match past the +// link cap, the count of near misses (--json lists them with their scores), or +// why nothing was compared. It prints nothing when no match was asked for. +func renderMatch(w io.Writer, o *match.Outcome) { + if o == nil { + return + } + if o.Skipped != "" { + fmt.Fprintf(w, " not matched: %s\n", termsafe.Sanitize(o.Skipped)) + } + for _, m := range o.Matches { + state := "link written" + if !m.Linked { + state = "listed, not linked" + } + fmt.Fprintf(w, " matched %s — %s (score %.3f, threshold %.2f): %s; confirm it by leaving the link, or remove the line\n", + termsafe.Sanitize(m.ID), m.Relation, m.Score, o.Threshold, state) + } + if o.Skipped == "" && len(o.Matches) == 0 { + fmt.Fprintf(w, " no match at threshold %.2f among %d record(s)", o.Threshold, o.Compared) + if n := len(o.NearMisses); n > 0 { + fmt.Fprintf(w, "; %d near miss(es) below it, listed with their scores by --json", n) + } + fmt.Fprintln(w) + } +} diff --git a/internal/surface/cli/match_cli_test.go b/internal/surface/cli/match_cli_test.go new file mode 100644 index 000000000..3c176d6bb --- /dev/null +++ b/internal/surface/cli/match_cli_test.go @@ -0,0 +1,145 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// The filing-time match on the front door (itd-2609212137116617): `capture` and +// the quoted-text `intent` create print each link they wrote, --json carries +// the whole outcome with the near misses and their scores, and a configuration +// the reader refuses files the record unlinked rather than refusing it. + +const ( + cliFinding = "The capture ledger reader silently skips a record whose frontmatter carries " + + "a duplicated key, so the finding disappears from every listing without a warning." + cliDouble = "Capture ledger reader silently skips any record whose frontmatter carries a " + + "duplicated key: the finding disappears from every listing, and no warning is printed." + cliFiller1 = "The site builder renders a stale anchor for a heading renamed since the last build." + cliFiller2 = "The history store drops a transcript that exceeds its byte budget without saying so." +) + +// matchRepo is a checkout with its own HOME, so the machine configuration +// layer is the fixture's, and two unrelated issues to weigh terms against. +func matchRepo(t *testing.T) string { + t.Helper() + repo := captureLedgerRepo(t) + t.Setenv("HOME", filepath.Join(t.TempDir(), "home")) + if err := os.MkdirAll(os.Getenv("HOME"), 0o700); err != nil { + t.Fatal(err) + } + runCLI(t, "capture", cliFiller1) + runCLI(t, "capture", cliFiller2) + return repo +} + +type captureJSON struct { + ID string `json:"id"` + Path string `json:"path"` + Match *struct { + Threshold float64 `json:"threshold"` + Skipped string `json:"skipped"` + Matches []struct { + ID string `json:"id"` + Relation string `json:"relation"` + Score float64 `json:"score"` + Linked bool `json:"linked"` + } `json:"matches"` + NearMisses []struct { + ID string `json:"id"` + Score float64 `json:"score"` + } `json:"near_misses"` + } `json:"match"` +} + +func captureJSONOf(t *testing.T, out []byte) captureJSON { + t.Helper() + var c captureJSON + if err := json.Unmarshal(out, &c); err != nil { + t.Fatalf("not JSON: %v\n%s", err, out) + } + return c +} + +func TestCaptureVerbLinksAndPrintsTheMatch(t *testing.T) { + repo := matchRepo(t) + first := captureJSONOf(t, runCLI(t, "capture", cliFinding, "--json")) + + out := string(runCLI(t, "capture", cliDouble)) + if !strings.Contains(out, "matched "+first.ID+" — duplicates") || !strings.Contains(out, "link written") { + t.Fatalf("the verb did not print the match against %s:\n%s", first.ID, out) + } + // The newest record is the double; it carries the link. + matches, _ := filepath.Glob(filepath.Join(repo, ".abcd/work/issues/open/iss-*.md")) + linked := 0 + for _, m := range matches { + b, _ := os.ReadFile(m) + if strings.Contains(string(b), "\nduplicates: ["+first.ID+"]\n") { + linked++ + } + } + if linked != 1 { + t.Fatalf("%d record(s) carry the duplicates link, want 1", linked) + } +} + +func TestCaptureVerbJSONListsNearMisses(t *testing.T) { + repo := matchRepo(t) + runCLI(t, "capture", cliFinding) + if err := os.WriteFile(filepath.Join(repo, ".abcd/config.json"), []byte(`{"match":{"threshold":0.99}}`), 0o644); err != nil { + t.Fatal(err) + } + c := captureJSONOf(t, runCLI(t, "capture", cliDouble, "--json")) + if c.Match == nil || c.Match.Threshold != 0.99 || len(c.Match.Matches) != 0 { + t.Fatalf("match = %+v, want nothing above the configured 0.99", c.Match) + } + if len(c.Match.NearMisses) == 0 || c.Match.NearMisses[0].Score <= 0 { + t.Fatalf("near misses = %+v, want the finding with its score", c.Match.NearMisses) + } + b, _ := os.ReadFile(filepath.Join(repo, c.Path)) + if strings.Contains(string(b), "duplicates:") || strings.Contains(string(b), "refines:") { + t.Fatalf("a link was written below the threshold:\n%s", b) + } +} + +func TestCaptureVerbFilesThroughARefusedConfiguration(t *testing.T) { + repo := matchRepo(t) + runCLI(t, "capture", cliFinding) + if err := os.WriteFile(filepath.Join(repo, ".abcd/config.json"), []byte(`{"match":{"treshold":0.5}}`), 0o644); err != nil { + t.Fatal(err) + } + out := string(runCLI(t, "capture", cliDouble)) + if !strings.Contains(out, "captured iss-") || !strings.Contains(out, "not matched") || !strings.Contains(out, "match.treshold") { + t.Fatalf("want the capture filed and the refused key named:\n%s", out) + } +} + +func TestIntentCreateVerbLinksAndPrintsTheMatch(t *testing.T) { + repo := matchRepo(t) + dir := filepath.Join(repo, ".abcd/development/intents/planned") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "itd-9-reader.md"), []byte("---\nid: itd-9\nslug: reader\nspec_id: null\nkind: null\n---\n\n# Ledger reader skips\n\n## Press Release\n\n> "+cliFinding+"\n"), 0o644); err != nil { + t.Fatal(err) + } + out := string(runCLI(t, "intent", cliDouble)) + if !strings.Contains(out, "created itd-") || !strings.Contains(out, "matched itd-9 — duplicates") { + t.Fatalf("the create did not print the match against itd-9:\n%s", out) + } + // Filed again, the text now doubles itd-9 and the draft just created. + c := captureJSONOf(t, runCLI(t, "intent", cliDouble+" Filed once more.", "--json")) + if c.Match == nil || len(c.Match.Matches) != 2 { + t.Fatalf("--json match = %+v, want itd-9 and the first draft", c.Match) + } + named := false + for _, m := range c.Match.Matches { + named = named || (m.ID == "itd-9" && m.Linked) + } + if !named { + t.Fatalf("--json match = %+v, want itd-9 linked", c.Match) + } +} From aafd4f2d0d8c30ab6409c3873cda551164205d2e Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:14:16 +0100 Subject: [PATCH 14/60] docs: the filing-time match on the capture and intent surfaces, and the itd-84 rung The capture and intent command pages, their brief chapters and the configuration chapter describe what ships: the match against every open and resolved issue and every intent, the duplicates/refines links, the near misses in --json, the never-refuse rule, and the match.threshold / match.fields keys the layered reader resolves from .abcd/config.json and ~/.abcd/config.json. The process explanation tells a reader what capture now decides for them. itd-84 marks its capture-time candidate pass delivered by itd-2609212137116617 (criterion 5), naming what stays undelivered; a test holds the mark. The reflect spec's embark ranking points at the canonical primitive rather than a score of its own in the lifeboat package. Assisted-by: Claude:claude-opus-5-5 --- .../brief/04-surfaces/05-intent.md | 8 +++- .../brief/04-surfaces/06-capture.md | 20 ++++++++++ .../brief/05-internals/03-configuration.md | 37 ++++++++++++----- .../itd-84-intent-decomposition.md | 11 +++++ .../spc-2609211751376504-reflect-command.md | 5 ++- commands/capture.md | 30 ++++++++++++++ commands/intent.md | 19 +++++++-- docs/explanation/process.md | 2 + internal/core/record/match/discipline_test.go | 40 +++++++++++++++++++ 9 files changed, 156 insertions(+), 16 deletions(-) create mode 100644 internal/core/record/match/discipline_test.go diff --git a/.abcd/development/brief/04-surfaces/05-intent.md b/.abcd/development/brief/04-surfaces/05-intent.md index 1a31c06fe..727cbbbaa 100644 --- a/.abcd/development/brief/04-surfaces/05-intent.md +++ b/.abcd/development/brief/04-surfaces/05-intent.md @@ -273,7 +273,7 @@ Later phase — intent-auditor (shape-classification role) scans the corpus | Subcommand | Purpose | File movement | |---|---|---| | `/abcd:intent` (no args) | Read-only status: bucket counts (drafts / planned / shipped / disciplines / superseded), open/closed spec counts, the itd↔spc links, a ledger-routing hint (`abcd capture "…"` for an observation, `abcd intent "…"` for a user-facing change), and an ideate-routing line (a big, unproven idea? `abcd ideate` runs the optional admission gauntlet and records the verdict either way) | — | -| `/abcd:intent ""` | **Canonical create** (spc-30 (predecessor store)/itd-46): a leading quoted seed is the canonical create entry. Seeds a draft skeleton whose `## Press Release` is the quoted text as prose, under an H1 derived from the text's first sentence (cut on a word boundary at the slug cap) or given as a title — one line, non-empty, redacted like the text — with Why This Matters and Acceptance Criteria seeded as prompts for the human to fill; assigns `itd-N` and derives the slug from the text; writes `suggested_kind: null`. An optional impact (additive, breaking or fix) stamps the draft's product impact at create time, and an optional production mode (hand-written, dictated-and-formatted or scribe-transcribed) stamps how its text was produced (itd-178); the draft's `origin` carries no flag and is derived from the verb that ran. A leading quote always creates — never falls through to bare render | writes to `drafts/itd-N-.md` (no spec created) | +| `/abcd:intent ""` | **Canonical create** (spc-30 (predecessor store)/itd-46): a leading quoted seed is the canonical create entry. Seeds a draft skeleton whose `## Press Release` is the quoted text as prose, under an H1 derived from the text's first sentence (cut on a word boundary at the slug cap) or given as a title — one line, non-empty, redacted like the text — with Why This Matters and Acceptance Criteria seeded as prompts for the human to fill; assigns `itd-N` and derives the slug from the text; writes `suggested_kind: null`. An optional impact (additive, breaking or fix) stamps the draft's product impact at create time, and an optional production mode (hand-written, dictated-and-formatted or scribe-transcribed) stamps how its text was produced (itd-178); the draft's `origin` carries no flag and is derived from the verb that ran. Before the draft is written, under the store's mint lock, its title and press release are matched against every open and resolved issue and every intent by the term-overlap heuristic `/abcd:capture` uses (itd-2609212137116617): a record at or above `match.threshold` is written onto the draft as `duplicates:` or `refines:`, at most three links, and the output lists the near misses below it with their scores; the match never refuses the create. A leading quote always creates — never falls through to bare render | writes to `drafts/itd-N-.md` (no spec created) | | The grill step, on one intent id | Socratic adversarial interview that stress-tests an intent for vagueness, missing acceptance, hidden assumptions before planning. Glossary-aware once `terminology/` exists. A brief-section mode would stress-test a brief section instead. (per itd-27, `intents/planned/` — a later phase; no grill sub-verb ships yet) | (stays in current state) | | Plan (one intent id) | Plans a draft: mints its native spec, injects the bidirectional link (intent `spec_id` ↔ spec `intent`), stamps an identity onto every unmarked scope condition, and moves the file `drafts/` → `planned/`. An impact given at planning stamps the INTENT's product-impact judgement, because the planning interview is where that judgement is made: validated at the create path's bar (never `internal`), written as the bare scalar the create path writes, refused before anything moves when it disagrees with a judgement the record already carries, and a no-op when it agrees; without one the field is left as found and the judgement stays owed to the close (iss-2609170726457256). A production mode given at planning stamps the MINTED SPEC's disclosure pair; the intent's own stamp was written at create time and is never rewritten. On an intent already in `planned/` it does the identity step alone (no spec, no move), takes an impact under the same rules, and refuses when nothing is unmarked and no judgement is added. Single intent ID. | `drafts/` → `planned/` (stamp step: no move) | | Readiness gate (one intent id, optionally with grounds) | **Implement-readiness gate**: reports whether an intent is ready to implement — eight checks, four of which gate: in `planned/`, with acceptance criteria, a bidirectional spec link, and a written spec body. The two claim rows (mechanism prompted-and-nullable, scope conditions with each condition identified) and the grounds row (a discipline record is exempt: it carries no conjecture of its own) are reported as advisory and never withhold readiness, their refusals parked by iss-2609091009111294 until the rethink of the reading work. The steps row is advisory by design: it reports the linked spec's `## Steps` shape — the steps listed and how many have landed, or none and so one step — and names a section that is not a numbered list with the shape it expects (itd-2609212103565953). Exit 0 ready / 1 not ready / 2 fault. Recording grounds, in the form `: `, is the gate's one write: it appends the conjecture behind this decision — what is expected, and what would show it wrong — to the intent's `## Grounds` section, append-only ([adr-57](../../decisions/adrs/0057-grounds-accumulate-as-an-append-only-section.md)), and then reports; a shipped or superseded record is never backfilled. | (no move; recorded grounds append to `## Grounds`) | @@ -326,6 +326,12 @@ production_mode: hand-written # how the text was produced: hand-written | dictat # draft, which graduated from nothing. The shipped # record_provenance rule holds it against origin: # extracted-from-record / contributed-by-reading +# duplicates: [iss-N | itd-N, ...] — written by the filing-time match (itd-2609212137116617) +# refines: [iss-N | itd-N, ...] when a record clears match.threshold: a near-identical +# double, or a broader record this draft is the narrower case of. +# A person confirms a link by leaving it and removes it by +# deleting the line; record_schema resolves both like every +# other cross-reference # Added later, not part of the seed skeleton: # held: "" — the hold the intent verb writes and its unhold removes, on a # drafts/ or planned/ record only: one non-empty line, redacted before the diff --git a/.abcd/development/brief/04-surfaces/06-capture.md b/.abcd/development/brief/04-surfaces/06-capture.md index 1705aefa8..a2915b265 100644 --- a/.abcd/development/brief/04-surfaces/06-capture.md +++ b/.abcd/development/brief/04-surfaces/06-capture.md @@ -78,6 +78,24 @@ record names no location in this checkout, so nothing ties it to the repository it is filed into: that is a nudge, not a gate, and it is the shape every misfiled record behind iss-2609120511058115 had (iss-2609231156260287). +Before the record is written, the fast path matches its text against the +record (itd-2609212137116617): every open and resolved issue's body and every +intent's title and press release, under the ledger lock, through the +term-overlap primitive in `internal/core/record/match`. The score is the share +of the new text's terms a candidate already holds, each term weighted by how +rare it is across the candidates, and it is declared a lexical heuristic on +every output. A candidate at or above `match.threshold` is written onto the +new record as `duplicates:` (the two hold each other's terms) or `refines:` +(the candidate holds this text's terms and more, so this record is the +narrower), at most three links; the output lists the rest, and the best five +below the threshold as near misses with their scores. The match never refuses +and never drops a capture: a text with fewer than eight distinct terms, a record +set that cannot be read and a configuration the reader refuses each file the +record unlinked, and the output says which. A person confirms a link by leaving +it and removes it by deleting its line, which leaves an ordinary record. The +match proposes no `reverses` and no `supersedes`: the itd-84 discipline keeps a +reversal advisory and human. + One flag belongs to one category: the lapse-instant flag carries the RFC 3339 instant a recorded discipline gave way, for the `lapse` category, and it has no default. @@ -255,6 +273,8 @@ related_specs: [spc-N, ...] related_issues: [iss-N, ...] synthesis_clusters: [