From 7664c830a2499a73012b14f615efe6b9ca13fd12 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Sat, 26 Sep 2026 00:15:13 +0200 Subject: [PATCH 1/2] prove: --pieces undoes each changed piece of code on its own --- README.md | 15 ++ cmd/test-check/main.go | 4 + cmd/test-check/prove.go | 44 +++++- internal/prove/pieces.go | 273 ++++++++++++++++++++++++++++++++++ internal/prove/pieces_test.go | 120 +++++++++++++++ internal/prove/prove.go | 66 ++++++-- skills/test-check/SKILL.md | 4 +- 7 files changed, 508 insertions(+), 18 deletions(-) create mode 100644 internal/prove/pieces.go create mode 100644 internal/prove/pieces_test.go diff --git a/README.md b/README.md index 5e6a06b..fc52e0b 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,21 @@ test-check prove --test "go test ./pkg -v -run '^{name}$'" Without `{name}`, the command runs once for the whole change, and one red test is enough. +### Every changed piece must be tested + +Red/green only checks the tests you wrote. `--pieces` checks the code you changed: + +```sh +test-check prove --list-tests # the tests this change adds or edits +test-check prove --pieces --test "go test ./pkg -run '^(TestA|TestB)$'" +``` + +It undoes one changed piece of code at a time (one diff hunk, a whole added file, or a whole deleted file) and runs the command. The command must fail every time. +- A piece that leaves the command passing is reported `not_tested`, and the outcome is `piece_not_tested` (exit 4). +- Comment- and whitespace-only hunks are skipped. +- Above `--max-pieces` (default 60), nothing runs and the outcome is `too_many_pieces`. +- Each piece goes through the same save-first stash, so `--restore` recovers an interrupted run. + Test files and fixtures keep their new versions for both runs. The reverted files are saved under the git dir before anything changes. An interrupted or crashed run is recovered with `test-check prove --restore`, and a new run refuses to start until then. Only one prove run may use a checkout at a time: a lock under the git dir makes a second run, or `--restore`, refuse while the first is alive. A lock left by a dead process is taken over. A test that hangs on the old code until `--timeout` counts as red. diff --git a/cmd/test-check/main.go b/cmd/test-check/main.go index 7c1ccea..cc3566e 100644 --- a/cmd/test-check/main.go +++ b/cmd/test-check/main.go @@ -60,6 +60,10 @@ func run(args []string, stdout, stderr io.Writer) int { run CMD on the new code (must pass), then with every non-test change reverted (must fail); files are restored afterwards. With {name} in CMD, every added or edited test must go red on its own + test-check prove --pieces --test CMD [--max-pieces N] [--dir DIR] [--base REF] [--json] + undo each changed piece of code on its own; CMD must fail every time + test-check prove --list-tests [--dir DIR] [--base REF] + print the Go/Python tests the change adds or edits test-check prove --restore [--dir DIR] put back files a crashed prove run left reverted diff --git a/cmd/test-check/prove.go b/cmd/test-check/prove.go index 0ea400c..868f2ce 100644 --- a/cmd/test-check/prove.go +++ b/cmd/test-check/prove.go @@ -8,6 +8,7 @@ import ( "io" "os" "os/signal" + "strconv" "strings" "syscall" "time" @@ -18,12 +19,14 @@ import ( const exitNotProven = 4 var proveMessages = map[string]string{ - prove.OutcomeProven: "The tests fail on the old code and pass on the new: they would catch a regression of this change.", - prove.OutcomeNotRedOnOld: "The tests PASS on the old code too, so they would not catch a regression. Make them fail without the fix.", - prove.OutcomeFailsOnNew: "The tests FAIL on the new code. Fix that before proving.", - prove.OutcomeNoTestChanged: "The change has no test files. If a test is needed, add one; otherwise record the one-off check you ran.", - prove.OutcomeNoCodeChanged: "Only tests changed; there is no old code to prove them against.", - prove.OutcomeNotRun: "A test never appeared in the command's output on the new code, so the command probably did not run it. Use a verbose runner (go test -v, pytest -v) and check the name.", + prove.OutcomeProven: "The tests fail on the old code and pass on the new: they would catch a regression of this change.", + prove.OutcomeNotRedOnOld: "The tests PASS on the old code too, so they would not catch a regression. Make them fail without the fix.", + prove.OutcomeFailsOnNew: "The tests FAIL on the new code. Fix that before proving.", + prove.OutcomeNoTestChanged: "The change has no test files. If a test is needed, add one; otherwise record the one-off check you ran.", + prove.OutcomeNoCodeChanged: "Only tests changed; there is no old code to prove them against.", + prove.OutcomePieceNotTested: "Undoing a changed piece of code left every test passing: no test notices it. Test that piece, or say why it needs none.", + prove.OutcomeTooManyPieces: "The change has more pieces than --max-pieces; nothing was run. Prove a narrower change or raise the limit.", + prove.OutcomeNotRun: "A test never appeared in the command's output on the new code, so the command probably did not run it. Use a verbose runner (go test -v, pytest -v) and check the name.", } func runProve(args []string, stdout, stderr io.Writer) int { @@ -35,6 +38,9 @@ func runProve(args []string, stdout, stderr io.Writer) int { each := fs.String("each", "", "comma-separated test names for {name} (default: the Go/Python tests the change adds or edits)") timeout := fs.Duration("timeout", 10*time.Minute, "limit for each test run") restore := fs.Bool("restore", false, "restore files a crashed prove run left reverted") + pieces := fs.Bool("pieces", false, "undo each changed piece of code on its own; every piece must turn the tests red") + maxPieces := fs.Int("max-pieces", prove.DefaultMaxPieces, "refuse piece mode above this many pieces") + listTests := fs.Bool("list-tests", false, "print the tests the change adds or edits, one per line, and exit") asJSON := fs.Bool("json", false, "print the report as JSON") if err := fs.Parse(args); err != nil { if err == flag.ErrHelp { @@ -42,6 +48,17 @@ func runProve(args []string, stdout, stderr io.Writer) int { } return exitUsage } + if *listTests { + names, err := prove.ListTests(context.Background(), *dir, *base) + if err != nil { + fmt.Fprintf(stderr, "test-check prove: %v\n", err) + return exitSource + } + for _, name := range names { + fmt.Fprintln(stdout, name) + } + return 0 + } if fs.NArg() != 0 || (*restore == (strings.TrimSpace(*test) != "")) { fmt.Fprintln(stderr, "test-check prove: pass exactly one of --test CMD or --restore") return exitUsage @@ -67,7 +84,7 @@ func runProve(args []string, stdout, stderr io.Writer) int { fmt.Fprintln(stderr, "test-check prove: --each needs {name} in --test") return exitUsage } - report, err := prove.Run(ctx, prove.Options{Dir: *dir, Base: *base, Test: *test, Each: names, Timeout: *timeout}) + report, err := prove.Run(ctx, prove.Options{Dir: *dir, Base: *base, Test: *test, Each: names, Timeout: *timeout, Pieces: *pieces, MaxPieces: *maxPieces}) if err != nil { fmt.Fprintf(stderr, "test-check prove: %v\n", err) return exitSource @@ -99,7 +116,18 @@ func runProve(args []string, stdout, stderr io.Writer) int { if len(report.RevertedFiles) > 0 && report.Outcome != prove.OutcomeNoTestChanged { fmt.Fprintf(stdout, "reverted for the old-code run: %s\n", strings.Join(report.RevertedFiles, ", ")) } - if len(report.Tests) > 0 { + for _, pc := range report.Pieces { + where := pc.File + if pc.Line > 0 { + where += ":" + strconv.Itoa(pc.Line) + } + note := "" + if pc.BuildErrorSuspected { + note = " (only a build error)" + } + fmt.Fprintf(stdout, " %-10s %s %s%s\n", pc.Outcome, where, pc.Snippet, note) + } + if len(report.Tests) > 0 || len(report.Pieces) > 0 { return code } switch report.Outcome { diff --git a/internal/prove/pieces.go b/internal/prove/pieces.go new file mode 100644 index 0000000..6895432 --- /dev/null +++ b/internal/prove/pieces.go @@ -0,0 +1,273 @@ +package prove + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/gitmoot/test-check/internal/check" +) + +const ( + // OutcomePieceNotTested: undoing one changed piece of code left every test + // passing, so no test notices that piece. + OutcomePieceNotTested = "piece_not_tested" + // OutcomeTooManyPieces: the change has more pieces than MaxPieces; nothing + // was run. Narrow the change or raise the limit. + OutcomeTooManyPieces = "too_many_pieces" + + PieceGuarded = "guarded" + PieceUnguarded = "not_tested" + + DefaultMaxPieces = 60 +) + +// PieceResult is one changed piece of code, undone on its own. +type PieceResult struct { + File string `json:"file"` + // Line is where the piece starts in the new file; 0 for a whole file. + Line int `json:"line"` + Lines int `json:"lines"` + Outcome string `json:"outcome"` + BuildErrorSuspected bool `json:"build_error_suspected"` + Snippet string `json:"snippet"` +} + +// piece is one hunk of a code file: undoing it means writing revert. +type piece struct { + file string + line int + lines int + snippet string + // revert is the file content with only this piece undone; exists=false + // means undoing it removes the file (the change added it). + revert []byte + exists bool + mode fs.FileMode + // fromBase: undo by writing the merge-base version (a deleted file). + fromBase bool + mergeBase string +} + +var hunkHeader = regexp.MustCompile(`^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@`) + +// commentOnly reports whether a changed line carries no behavior: blank, or a +// comment in one of the languages test-check knows. +func commentOnly(line string) bool { + t := strings.TrimSpace(line) + for _, prefix := range []string{"//", "#", "/*", "*", "--", `"""`, "'''"} { + if strings.HasPrefix(t, prefix) { + return true + } + } + return t == "" +} + +// codePieces splits the change to each code file into hunks. Comment- and +// whitespace-only hunks are skipped: no test can notice them. +func codePieces(r repo, mergeBase string, files []string) ([]piece, error) { + var pieces []piece + for _, p := range files { + if !check.IsCode(p) { + continue + } + full := filepath.Join(r.root, p) + info, statErr := os.Lstat(full) + current, readErr := os.ReadFile(full) + listing, err := r.git("ls-tree", mergeBase, "--", p) + if err != nil { + return nil, err + } + switch { + case strings.TrimSpace(listing) == "": + // Added by the change: the whole file is one piece. + if readErr != nil { + continue + } + pieces = append(pieces, piece{file: p, lines: strings.Count(string(current), "\n"), snippet: "(new file)", exists: false}) + continue + case errors.Is(statErr, fs.ErrNotExist): + // Deleted by the change: undoing it restores the base file. + pieces = append(pieces, piece{file: p, snippet: "(deleted file)", fromBase: true, mergeBase: mergeBase}) + continue + case statErr != nil: + return nil, statErr + case readErr != nil: + return nil, readErr + } + if info.Mode()&fs.ModeSymlink != 0 { + continue + } + diff, err := r.git("diff", "-U0", "--no-color", "--no-ext-diff", mergeBase, "--", p) + if err != nil { + return nil, err + } + hunks, err := fileHunks(string(current), diff) + if err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + for _, h := range hunks { + h.file, h.exists, h.mode = p, true, info.Mode().Perm() + pieces = append(pieces, h) + } + } + return pieces, nil +} + +// fileHunks parses a -U0 diff of one file and builds, for each behavioral +// hunk, the current content with only that hunk undone. +func fileHunks(current, diff string) ([]piece, error) { + lines := strings.SplitAfter(current, "\n") + if len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + var pieces []piece + diffLines := strings.Split(diff, "\n") + for i := 0; i < len(diffLines); i++ { + m := hunkHeader.FindStringSubmatch(diffLines[i]) + if m == nil { + continue + } + newStart, _ := strconv.Atoi(m[3]) + newCount := 1 + if m[4] != "" { + newCount, _ = strconv.Atoi(m[4]) + } + var oldLines []string + behavioral := false + snippet := "" + newSnippet := false + for i+1 < len(diffLines) { + next := diffLines[i+1] + if next == "" || (next[0] != '+' && next[0] != '-' && next[0] != '\\') { + break + } + i++ + switch next[0] { + case '-': + oldLines = append(oldLines, next[1:]+"\n") + case '\\': + // "\ No newline at end of file": drop it from the old side. + if n := len(oldLines); n > 0 && strings.HasPrefix(diffLines[i-1], "-") { + oldLines[n-1] = strings.TrimSuffix(oldLines[n-1], "\n") + } + continue + } + if !commentOnly(next[1:]) { + behavioral = true + // Show the first new line; a pure deletion shows the removed one. + if next[0] == '+' && !newSnippet { + snippet, newSnippet = strings.TrimSpace(next[1:]), true + } else if snippet == "" { + snippet = "(removed) " + strings.TrimSpace(next[1:]) + } + } + } + if !behavioral { + continue + } + at := newStart - 1 + if newCount == 0 { + at = newStart + } + if at < 0 || at+newCount > len(lines) { + return nil, fmt.Errorf("hunk at +%d,%d outside a %d-line file", newStart, newCount, len(lines)) + } + reverted := make([]string, 0, len(lines)-newCount+len(oldLines)) + reverted = append(reverted, lines[:at]...) + reverted = append(reverted, oldLines...) + reverted = append(reverted, lines[at+newCount:]...) + if len(snippet) > 120 { + snippet = strings.ToValidUTF8(snippet[:120], "") + } + pieces = append(pieces, piece{line: newStart, lines: newCount, snippet: snippet, revert: []byte(strings.Join(reverted, ""))}) + } + return pieces, nil +} + +// applyPiece stashes the file, then writes it with only this piece undone. +func applyPiece(r repo, pc piece) error { + if err := stashPaths(r, []string{pc.file}); err != nil { + return err + } + full := filepath.Join(r.root, pc.file) + if pc.fromBase { + return writeBase(r, pc.mergeBase, pc.file) + } + if !pc.exists { + return os.RemoveAll(full) + } + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return err + } + _ = os.Remove(full) + return os.WriteFile(full, pc.revert, pc.mode) +} + +// runPieces undoes each changed piece of code on its own and runs the test +// command: every piece must turn some test red. +func runPieces(ctx context.Context, r repo, mergeBase string, opts Options, report Report) (Report, error) { + pieces, err := codePieces(r, mergeBase, report.RevertedFiles) + if err != nil { + return Report{}, err + } + limit := opts.MaxPieces + if limit <= 0 { + limit = DefaultMaxPieces + } + if len(pieces) > limit { + report.Outcome = OutcomeTooManyPieces + for _, pc := range pieces { + report.Pieces = append(report.Pieces, PieceResult{File: pc.file, Line: pc.line, Lines: pc.lines, Snippet: pc.snippet}) + } + return report, nil + } + if len(pieces) == 0 { + report.Outcome = OutcomeNoCodeChanged + return report, nil + } + newOut, newErr := runTest(ctx, r.root, opts.Test, opts.Timeout) + report.NewOutput = tail(newOut) + if ctx.Err() != nil { + return Report{}, ctx.Err() + } + if newErr != nil { + report.Outcome = OutcomeFailsOnNew + return report, nil + } + report.Outcome = OutcomeProven + for _, pc := range pieces { + res := PieceResult{File: pc.file, Line: pc.line, Lines: pc.lines, Snippet: pc.snippet} + if err := applyPiece(r, pc); err != nil { + if errors.Is(err, errStashTaken) { + return Report{}, err + } + if restoreErr := Restore(context.WithoutCancel(ctx), r.root); restoreErr != nil { + return Report{}, fmt.Errorf("undo piece %s:%d failed: %v; restore also failed: %w", pc.file, pc.line, err, restoreErr) + } + return Report{}, fmt.Errorf("undo piece %s:%d failed (files restored): %w", pc.file, pc.line, err) + } + out, testErr := runTest(ctx, r.root, opts.Test, opts.Timeout) + if err := Restore(context.WithoutCancel(ctx), r.root); err != nil { + return Report{}, fmt.Errorf("restore failed; recover with `test-check prove --restore`: %w", err) + } + if ctx.Err() != nil { + return Report{}, ctx.Err() + } + if testErr == nil { + res.Outcome = PieceUnguarded + report.Outcome = OutcomePieceNotTested + } else { + res.Outcome = PieceGuarded + res.BuildErrorSuspected = buildError.MatchString(out) + } + report.Pieces = append(report.Pieces, res) + } + return report, nil +} diff --git a/internal/prove/pieces_test.go b/internal/prove/pieces_test.go new file mode 100644 index 0000000..737db92 --- /dev/null +++ b/internal/prove/pieces_test.go @@ -0,0 +1,120 @@ +package prove + +import ( + "context" + "os/exec" + "reflect" + "strings" + "testing" +) + +// The hunk arithmetic decides what "undo one piece" writes, so check it +// against git's own -U0 diffs for replacement, pure insertion, pure deletion, +// and a missing final newline. +func TestFileHunksUndoExactlyOneHunk(t *testing.T) { + cases := []struct{ name, base, current string }{ + {"replace", "a\nb\nc\nd\n", "a\nB\nc\nd\n"}, + {"insert", "a\nb\nc\n", "a\nb\nx\ny\nc\n"}, + {"delete", "a\nb\nc\nd\n", "a\nd\n"}, + {"insert at start", "a\nb\n", "z\na\nb\n"}, + {"delete at end", "a\nb\nc\n", "a\n"}, + {"no final newline", "a\nb", "a\nB"}, + {"two hunks", "a\nb\nc\nd\ne\nf\n", "A\nb\nc\nd\ne\nF\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + git(t, dir, "init", "-q", "-b", "main") + put(t, dir, "f.go", tc.base) + git(t, dir, "add", ".") + git(t, dir, "commit", "-q", "-m", "base") + put(t, dir, "f.go", tc.current) + out, err := exec.Command("git", "-C", dir, "diff", "-U0", "--no-color", "HEAD", "--", "f.go").Output() + if err != nil { + t.Fatal(err) + } + hunks, err := fileHunks(tc.current, string(out)) + if err != nil { + t.Fatal(err) + } + if tc.name == "two hunks" { + // Undoing each hunk alone keeps the other one. + want := []string{"a\nb\nc\nd\ne\nF\n", "A\nb\nc\nd\ne\nf\n"} + if len(hunks) != 2 || string(hunks[0].revert) != want[0] || string(hunks[1].revert) != want[1] { + t.Fatalf("reverts = %q", hunkReverts(hunks)) + } + return + } + if len(hunks) != 1 || string(hunks[0].revert) != tc.base { + t.Fatalf("reverts = %q, want [%q]", hunkReverts(hunks), tc.base) + } + }) + } +} + +func hunkReverts(hunks []piece) []string { + var out []string + for _, h := range hunks { + out = append(out, string(h.revert)) + } + return out +} + +// piecesFixture: the fix changes Stop (tested) and Clamp (never tested) and +// edits a comment. Only Clamp's piece must be reported. +func piecesFixture(t *testing.T) string { + t.Helper() + t.Setenv("GOTOOLCHAIN", "local") + t.Setenv("GOFLAGS", "-buildvcs=false") + dir := t.TempDir() + git(t, dir, "init", "-q", "-b", "main") + put(t, dir, "go.mod", "module example.com/stop\n\ngo 1.22\n") + code := "package stop\n\n// Stop says when to stop.\nfunc Stop(n int) bool { return n > 3 }\n\nfunc Clamp(n int) int {\n\tif n > 10 {\n\t\treturn 10\n\t}\n\treturn n\n}\n" + put(t, dir, "stop.go", code) + git(t, dir, "add", ".") + git(t, dir, "commit", "-q", "-m", "base") + git(t, dir, "checkout", "-q", "-b", "fix") + code = strings.Replace(code, "// Stop says when to stop.", "// Stop reports whether n reached the limit.", 1) + code = strings.Replace(code, "n > 3", "n >= 3", 1) + code = strings.Replace(code, "if n > 10", "if n >= 10", 1) + put(t, dir, "stop.go", code) + put(t, dir, "stop_test.go", "package stop\n\nimport \"testing\"\n\nfunc TestStopAtThree(t *testing.T) {\n\tif !Stop(3) {\n\t\tt.Fatal(\"3\")\n\t}\n}\n") + return dir +} + +func TestPiecesReportsTheUntestedPieceAndRestores(t *testing.T) { + dir := piecesFixture(t) + before := snapshot(t, dir) + report, err := Run(context.Background(), Options{Dir: dir, Base: "main", Test: "go test -count=1 .", Pieces: true}) + if err != nil { + t.Fatal(err) + } + var got []string + for _, pc := range report.Pieces { + got = append(got, pc.Outcome+" "+pc.Snippet) + } + want := []string{"guarded func Stop(n int) bool { return n >= 3 }", "not_tested if n >= 10 {"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("pieces = %q, want %q (the comment-only hunk must be skipped)", got, want) + } + if report.Outcome != OutcomePieceNotTested { + t.Fatalf("outcome = %s", report.Outcome) + } + if after := snapshot(t, dir); !reflect.DeepEqual(before, after) { + t.Fatalf("files changed by prove:\nbefore %v\nafter %v", before, after) + } +} + +func TestPiecesRefusesAboveTheLimitWithoutRunning(t *testing.T) { + dir := piecesFixture(t) + report, err := Run(context.Background(), Options{Dir: dir, Base: "main", Test: "touch ran && false", Pieces: true, MaxPieces: 1}) + if err != nil { + t.Fatal(err) + } + if report.Outcome != OutcomeTooManyPieces || len(report.Pieces) != 2 { + t.Fatalf("report = %+v", report) + } + if _, ok := snapshot(t, dir)["ran"]; ok { + t.Fatal("test command ran above the piece limit") + } +} diff --git a/internal/prove/prove.go b/internal/prove/prove.go index 21e79f8..74a0a0c 100644 --- a/internal/prove/prove.go +++ b/internal/prove/prove.go @@ -57,6 +57,8 @@ type Report struct { // Tests holds one result per test in per-test mode. The overall Outcome is // proven only when every test is. Tests []TestResult `json:"tests,omitempty"` + // Pieces holds one result per changed piece of code in piece mode. + Pieces []PieceResult `json:"pieces,omitempty"` } // TestResult is one test's red/green result in per-test mode. @@ -78,6 +80,10 @@ type Options struct { Test string Each []string Timeout time.Duration + // Pieces undoes each changed piece of code on its own instead of all of + // it at once; every piece must turn the test command red. + Pieces bool + MaxPieces int } var testSupport = regexp.MustCompile(`(^|/)(testdata|fixtures?|__snapshots__|__fixtures__)/`) @@ -158,6 +164,12 @@ func Run(ctx context.Context, opts Options) (Report, error) { return report, nil } + if opts.Pieces { + if strings.Contains(opts.Test, NamePlaceholder) { + return Report{}, errors.New("piece mode runs the test command as given; drop {name}") + } + return runPieces(ctx, r, mergeBase, opts, report) + } if !strings.Contains(opts.Test, NamePlaceholder) { return runWhole(ctx, r, mergeBase, opts, report) } @@ -382,6 +394,20 @@ type entry struct { // the manifest BEFORE touching the working tree, then puts each path back to // its base version (deleting it if the base did not have it). func swapToBase(r repo, mergeBase string, paths []string) error { + if err := stashPaths(r, paths); err != nil { + return err + } + for _, p := range paths { + if err := writeBase(r, mergeBase, p); err != nil { + return err + } + } + return nil +} + +// stashPaths saves the current version of each path under the git dir and +// writes the manifest, synced, before the caller touches the working tree. +func stashPaths(r repo, paths []string) error { stash := filepath.Join(r.gitDir, stashDir) if err := os.Mkdir(stash, 0o700); err != nil { if errors.Is(err, fs.ErrExist) { @@ -420,15 +446,7 @@ func swapToBase(r repo, mergeBase string, paths []string) error { if err != nil { return err } - if err := writeSynced(filepath.Join(stash, manifestName), manifest); err != nil { - return err - } - for _, p := range paths { - if err := writeBase(r, mergeBase, p); err != nil { - return err - } - } - return nil + return writeSynced(filepath.Join(stash, manifestName), manifest) } // writeBase puts one path back to its merge-base version: a symlink as a @@ -603,3 +621,33 @@ func lockHolder(path string) int { pid, _ := strconv.Atoi(strings.TrimSpace(string(raw))) return pid } + +// ListTests returns the Go and Python tests the change adds or edits, the same +// names per-test mode would run. +func ListTests(ctx context.Context, dir, base string) ([]string, error) { + r, err := open(ctx, dir) + if err != nil { + return nil, err + } + if base == "" { + if base, err = source.DefaultBase(r.git); err != nil { + return nil, err + } + } + mergeBase, err := r.git("merge-base", base, "HEAD") + if err != nil { + return nil, err + } + mergeBase = strings.TrimSpace(mergeBase) + changed, err := changedPaths(r, mergeBase) + if err != nil { + return nil, err + } + var tests []string + for _, p := range changed { + if IsTestSide(p) { + tests = append(tests, p) + } + } + return ChangedTestNames(r.git, mergeBase, r.root, tests) +} diff --git a/skills/test-check/SKILL.md b/skills/test-check/SKILL.md index a56d08f..8bbb9a6 100644 --- a/skills/test-check/SKILL.md +++ b/skills/test-check/SKILL.md @@ -29,7 +29,9 @@ A permanent test earns its place only if it protects behavior someone relies on | `no_test_in_change` | 4 | No test file changed. Add one, or treat it as a one-off check. | | `test_not_run` | 4 | The command never mentioned the test, so it probably selected nothing. Fix the command or the name. | - A `build_error_suspected` warning means the old run failed only because the test uses new names; for a bug fix, test through an interface the old code already had. If a run is interrupted, `test-check prove --restore` puts your files back. For a one-off check: run it and keep the output. Done when the PR description records the `prove` outcome (or the one-off command and what you observed) — not before. + A `build_error_suspected` warning means the old run failed only because the test uses new names; for a bug fix, test through an interface the old code already had. If a run is interrupted, `test-check prove --restore` puts your files back. For a one-off check: run it and keep the output. + + Then check that no changed code is left untested: `test-check prove --pieces --test ""` (`test-check prove --list-tests` prints them). It undoes each changed piece of code on its own and expects the tests to fail each time. A `not_tested` piece is behavior nothing checks: test it, or say in the PR why it needs no test. Done when the PR description records both `prove` outcomes (or the one-off command and what you observed) — not before. `test-check` is advice. Your own evidence decides; never skip verification because it said no test. From 0668968831002b3b7afbd4f6d034004c3d5fd02a Mon Sep 17 00:00:00 2001 From: jerryfane Date: Sat, 26 Sep 2026 00:18:46 +0200 Subject: [PATCH 2/2] Keep the piece check advisory: it failed its noise bar --- README.md | 4 +++- skills/test-check/SKILL.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fc52e0b..59968be 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ test-check prove --test "go test ./pkg -v -run '^{name}$'" Without `{name}`, the command runs once for the whole change, and one red test is enough. -### Every changed piece must be tested +### Which changed pieces no test checks (advisory) Red/green only checks the tests you wrote. `--pieces` checks the code you changed: @@ -54,6 +54,8 @@ It undoes one changed piece of code at a time (one diff hunk, a whole added file - Above `--max-pieces` (default 60), nothing runs and the outcome is `too_many_pieces`. - Each piece goes through the same save-first stash, so `--restore` recovers an interrupted run. +Treat the list as advice. On 20 past PRs that reviewers approved without test findings, 15 had at least one `not_tested` piece: often logging, wording or defaults that need no test. + Test files and fixtures keep their new versions for both runs. The reverted files are saved under the git dir before anything changes. An interrupted or crashed run is recovered with `test-check prove --restore`, and a new run refuses to start until then. Only one prove run may use a checkout at a time: a lock under the git dir makes a second run, or `--restore`, refuse while the first is alive. A lock left by a dead process is taken over. A test that hangs on the old code until `--timeout` counts as red. diff --git a/skills/test-check/SKILL.md b/skills/test-check/SKILL.md index 8bbb9a6..cf5348f 100644 --- a/skills/test-check/SKILL.md +++ b/skills/test-check/SKILL.md @@ -31,7 +31,7 @@ A permanent test earns its place only if it protects behavior someone relies on A `build_error_suspected` warning means the old run failed only because the test uses new names; for a bug fix, test through an interface the old code already had. If a run is interrupted, `test-check prove --restore` puts your files back. For a one-off check: run it and keep the output. - Then check that no changed code is left untested: `test-check prove --pieces --test ""` (`test-check prove --list-tests` prints them). It undoes each changed piece of code on its own and expects the tests to fail each time. A `not_tested` piece is behavior nothing checks: test it, or say in the PR why it needs no test. Done when the PR description records both `prove` outcomes (or the one-off command and what you observed) — not before. + Optionally, look for changed code no test checks: `test-check prove --pieces --test ""` (`test-check prove --list-tests` prints them). It undoes each changed piece of code on its own and lists every `not_tested` piece. Many are fine (logging, wording, defaults); use the list to decide what deserves a test, not as a pass/fail. Done when the PR description records the `prove` outcome (or the one-off command and what you observed) — not before. `test-check` is advice. Your own evidence decides; never skip verification because it said no test.