Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,23 @@ test-check prove --test "go test ./pkg -v -run '^{name}$'"

Without `{name}`, the command runs once for the whole change, and one red test is enough.

### Which changed pieces no test checks (advisory)

Red/green only checks the tests you wrote. `--pieces` checks the code you changed:

```sh
test-check prove --list-tests # the tests this change adds or edits
test-check prove --pieces --test "go test ./pkg -run '^(TestA|TestB)$'"
```

It undoes one changed piece of code at a time (one diff hunk, a whole added file, or a whole deleted file) and runs the command. The command must fail every time.
- A piece that leaves the command passing is reported `not_tested`, and the outcome is `piece_not_tested` (exit 4).
- Comment- and whitespace-only hunks are skipped.
- Above `--max-pieces` (default 60), nothing runs and the outcome is `too_many_pieces`.
- Each piece goes through the same save-first stash, so `--restore` recovers an interrupted run.

Treat the list as advice. On 20 past PRs that reviewers approved without test findings, 15 had at least one `not_tested` piece: often logging, wording or defaults that need no test.

Test files and fixtures keep their new versions for both runs. The reverted files are saved under the git dir before anything changes. An interrupted or crashed run is recovered with `test-check prove --restore`, and a new run refuses to start until then.

Only one prove run may use a checkout at a time: a lock under the git dir makes a second run, or `--restore`, refuse while the first is alive. A lock left by a dead process is taken over. A test that hangs on the old code until `--timeout` counts as red.
Expand Down
4 changes: 4 additions & 0 deletions cmd/test-check/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@ func run(args []string, stdout, stderr io.Writer) int {
run CMD on the new code (must pass), then with every non-test change
reverted (must fail); files are restored afterwards. With {name} in
CMD, every added or edited test must go red on its own
test-check prove --pieces --test CMD [--max-pieces N] [--dir DIR] [--base REF] [--json]
undo each changed piece of code on its own; CMD must fail every time
test-check prove --list-tests [--dir DIR] [--base REF]
print the Go/Python tests the change adds or edits
test-check prove --restore [--dir DIR]
put back files a crashed prove run left reverted

Expand Down
44 changes: 36 additions & 8 deletions cmd/test-check/prove.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"io"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
Expand All @@ -18,12 +19,14 @@ import (
const exitNotProven = 4

var proveMessages = map[string]string{
prove.OutcomeProven: "The tests fail on the old code and pass on the new: they would catch a regression of this change.",
prove.OutcomeNotRedOnOld: "The tests PASS on the old code too, so they would not catch a regression. Make them fail without the fix.",
prove.OutcomeFailsOnNew: "The tests FAIL on the new code. Fix that before proving.",
prove.OutcomeNoTestChanged: "The change has no test files. If a test is needed, add one; otherwise record the one-off check you ran.",
prove.OutcomeNoCodeChanged: "Only tests changed; there is no old code to prove them against.",
prove.OutcomeNotRun: "A test never appeared in the command's output on the new code, so the command probably did not run it. Use a verbose runner (go test -v, pytest -v) and check the name.",
prove.OutcomeProven: "The tests fail on the old code and pass on the new: they would catch a regression of this change.",
prove.OutcomeNotRedOnOld: "The tests PASS on the old code too, so they would not catch a regression. Make them fail without the fix.",
prove.OutcomeFailsOnNew: "The tests FAIL on the new code. Fix that before proving.",
prove.OutcomeNoTestChanged: "The change has no test files. If a test is needed, add one; otherwise record the one-off check you ran.",
prove.OutcomeNoCodeChanged: "Only tests changed; there is no old code to prove them against.",
prove.OutcomePieceNotTested: "Undoing a changed piece of code left every test passing: no test notices it. Test that piece, or say why it needs none.",
prove.OutcomeTooManyPieces: "The change has more pieces than --max-pieces; nothing was run. Prove a narrower change or raise the limit.",
prove.OutcomeNotRun: "A test never appeared in the command's output on the new code, so the command probably did not run it. Use a verbose runner (go test -v, pytest -v) and check the name.",
}

func runProve(args []string, stdout, stderr io.Writer) int {
Expand All @@ -35,13 +38,27 @@ func runProve(args []string, stdout, stderr io.Writer) int {
each := fs.String("each", "", "comma-separated test names for {name} (default: the Go/Python tests the change adds or edits)")
timeout := fs.Duration("timeout", 10*time.Minute, "limit for each test run")
restore := fs.Bool("restore", false, "restore files a crashed prove run left reverted")
pieces := fs.Bool("pieces", false, "undo each changed piece of code on its own; every piece must turn the tests red")
maxPieces := fs.Int("max-pieces", prove.DefaultMaxPieces, "refuse piece mode above this many pieces")
listTests := fs.Bool("list-tests", false, "print the tests the change adds or edits, one per line, and exit")
asJSON := fs.Bool("json", false, "print the report as JSON")
if err := fs.Parse(args); err != nil {
if err == flag.ErrHelp {
return 0
}
return exitUsage
}
if *listTests {
names, err := prove.ListTests(context.Background(), *dir, *base)
if err != nil {
fmt.Fprintf(stderr, "test-check prove: %v\n", err)
return exitSource
}
for _, name := range names {
fmt.Fprintln(stdout, name)
}
return 0
}
if fs.NArg() != 0 || (*restore == (strings.TrimSpace(*test) != "")) {
fmt.Fprintln(stderr, "test-check prove: pass exactly one of --test CMD or --restore")
return exitUsage
Expand All @@ -67,7 +84,7 @@ func runProve(args []string, stdout, stderr io.Writer) int {
fmt.Fprintln(stderr, "test-check prove: --each needs {name} in --test")
return exitUsage
}
report, err := prove.Run(ctx, prove.Options{Dir: *dir, Base: *base, Test: *test, Each: names, Timeout: *timeout})
report, err := prove.Run(ctx, prove.Options{Dir: *dir, Base: *base, Test: *test, Each: names, Timeout: *timeout, Pieces: *pieces, MaxPieces: *maxPieces})
if err != nil {
fmt.Fprintf(stderr, "test-check prove: %v\n", err)
return exitSource
Expand Down Expand Up @@ -99,7 +116,18 @@ func runProve(args []string, stdout, stderr io.Writer) int {
if len(report.RevertedFiles) > 0 && report.Outcome != prove.OutcomeNoTestChanged {
fmt.Fprintf(stdout, "reverted for the old-code run: %s\n", strings.Join(report.RevertedFiles, ", "))
}
if len(report.Tests) > 0 {
for _, pc := range report.Pieces {
where := pc.File
if pc.Line > 0 {
where += ":" + strconv.Itoa(pc.Line)
}
note := ""
if pc.BuildErrorSuspected {
note = " (only a build error)"
}
fmt.Fprintf(stdout, " %-10s %s %s%s\n", pc.Outcome, where, pc.Snippet, note)
}
if len(report.Tests) > 0 || len(report.Pieces) > 0 {
return code
}
switch report.Outcome {
Expand Down
273 changes: 273 additions & 0 deletions internal/prove/pieces.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,273 @@
package prove

import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"

"github.com/gitmoot/test-check/internal/check"
)

const (
// OutcomePieceNotTested: undoing one changed piece of code left every test
// passing, so no test notices that piece.
OutcomePieceNotTested = "piece_not_tested"
// OutcomeTooManyPieces: the change has more pieces than MaxPieces; nothing
// was run. Narrow the change or raise the limit.
OutcomeTooManyPieces = "too_many_pieces"

PieceGuarded = "guarded"
PieceUnguarded = "not_tested"

DefaultMaxPieces = 60
)

// PieceResult is one changed piece of code, undone on its own.
type PieceResult struct {
File string `json:"file"`
// Line is where the piece starts in the new file; 0 for a whole file.
Line int `json:"line"`
Lines int `json:"lines"`
Outcome string `json:"outcome"`
BuildErrorSuspected bool `json:"build_error_suspected"`
Snippet string `json:"snippet"`
}

// piece is one hunk of a code file: undoing it means writing revert.
type piece struct {
file string
line int
lines int
snippet string
// revert is the file content with only this piece undone; exists=false
// means undoing it removes the file (the change added it).
revert []byte
exists bool
mode fs.FileMode
// fromBase: undo by writing the merge-base version (a deleted file).
fromBase bool
mergeBase string
}

var hunkHeader = regexp.MustCompile(`^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@`)

// commentOnly reports whether a changed line carries no behavior: blank, or a
// comment in one of the languages test-check knows.
func commentOnly(line string) bool {
t := strings.TrimSpace(line)
for _, prefix := range []string{"//", "#", "/*", "*", "--", `"""`, "'''"} {
if strings.HasPrefix(t, prefix) {
return true
}
}
return t == ""
}

// codePieces splits the change to each code file into hunks. Comment- and
// whitespace-only hunks are skipped: no test can notice them.
func codePieces(r repo, mergeBase string, files []string) ([]piece, error) {
var pieces []piece
for _, p := range files {
if !check.IsCode(p) {
continue
}
full := filepath.Join(r.root, p)
info, statErr := os.Lstat(full)
current, readErr := os.ReadFile(full)
listing, err := r.git("ls-tree", mergeBase, "--", p)
if err != nil {
return nil, err
}
switch {
case strings.TrimSpace(listing) == "":
// Added by the change: the whole file is one piece.
if readErr != nil {
continue
}
pieces = append(pieces, piece{file: p, lines: strings.Count(string(current), "\n"), snippet: "(new file)", exists: false})
continue
case errors.Is(statErr, fs.ErrNotExist):
// Deleted by the change: undoing it restores the base file.
pieces = append(pieces, piece{file: p, snippet: "(deleted file)", fromBase: true, mergeBase: mergeBase})
continue
case statErr != nil:
return nil, statErr
case readErr != nil:
return nil, readErr
}
if info.Mode()&fs.ModeSymlink != 0 {
continue
}
diff, err := r.git("diff", "-U0", "--no-color", "--no-ext-diff", mergeBase, "--", p)
if err != nil {
return nil, err
}
hunks, err := fileHunks(string(current), diff)
if err != nil {
return nil, fmt.Errorf("%s: %w", p, err)
}
for _, h := range hunks {
h.file, h.exists, h.mode = p, true, info.Mode().Perm()
pieces = append(pieces, h)
}
}
return pieces, nil
}

// fileHunks parses a -U0 diff of one file and builds, for each behavioral
// hunk, the current content with only that hunk undone.
func fileHunks(current, diff string) ([]piece, error) {
lines := strings.SplitAfter(current, "\n")
if len(lines) > 0 && lines[len(lines)-1] == "" {
lines = lines[:len(lines)-1]
}
var pieces []piece
diffLines := strings.Split(diff, "\n")
for i := 0; i < len(diffLines); i++ {
m := hunkHeader.FindStringSubmatch(diffLines[i])
if m == nil {
continue
}
newStart, _ := strconv.Atoi(m[3])
newCount := 1
if m[4] != "" {
newCount, _ = strconv.Atoi(m[4])
}
var oldLines []string
behavioral := false
snippet := ""
newSnippet := false
for i+1 < len(diffLines) {
next := diffLines[i+1]
if next == "" || (next[0] != '+' && next[0] != '-' && next[0] != '\\') {
break
}
i++
switch next[0] {
case '-':
oldLines = append(oldLines, next[1:]+"\n")
case '\\':
// "\ No newline at end of file": drop it from the old side.
if n := len(oldLines); n > 0 && strings.HasPrefix(diffLines[i-1], "-") {
oldLines[n-1] = strings.TrimSuffix(oldLines[n-1], "\n")
}
continue
}
if !commentOnly(next[1:]) {
behavioral = true
// Show the first new line; a pure deletion shows the removed one.
if next[0] == '+' && !newSnippet {
snippet, newSnippet = strings.TrimSpace(next[1:]), true
} else if snippet == "" {
snippet = "(removed) " + strings.TrimSpace(next[1:])
}
}
}
if !behavioral {
continue
}
at := newStart - 1
if newCount == 0 {
at = newStart
}
if at < 0 || at+newCount > len(lines) {
return nil, fmt.Errorf("hunk at +%d,%d outside a %d-line file", newStart, newCount, len(lines))
}
reverted := make([]string, 0, len(lines)-newCount+len(oldLines))
reverted = append(reverted, lines[:at]...)
reverted = append(reverted, oldLines...)
reverted = append(reverted, lines[at+newCount:]...)
if len(snippet) > 120 {
snippet = strings.ToValidUTF8(snippet[:120], "")
}
pieces = append(pieces, piece{line: newStart, lines: newCount, snippet: snippet, revert: []byte(strings.Join(reverted, ""))})
}
return pieces, nil
}

// applyPiece stashes the file, then writes it with only this piece undone.
func applyPiece(r repo, pc piece) error {
if err := stashPaths(r, []string{pc.file}); err != nil {
return err
}
full := filepath.Join(r.root, pc.file)
if pc.fromBase {
return writeBase(r, pc.mergeBase, pc.file)
}
if !pc.exists {
return os.RemoveAll(full)
}
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
return err
}
_ = os.Remove(full)
return os.WriteFile(full, pc.revert, pc.mode)
}

// runPieces undoes each changed piece of code on its own and runs the test
// command: every piece must turn some test red.
func runPieces(ctx context.Context, r repo, mergeBase string, opts Options, report Report) (Report, error) {
pieces, err := codePieces(r, mergeBase, report.RevertedFiles)
if err != nil {
return Report{}, err
}
limit := opts.MaxPieces
if limit <= 0 {
limit = DefaultMaxPieces
}
if len(pieces) > limit {
report.Outcome = OutcomeTooManyPieces
for _, pc := range pieces {
report.Pieces = append(report.Pieces, PieceResult{File: pc.file, Line: pc.line, Lines: pc.lines, Snippet: pc.snippet})
}
return report, nil
}
if len(pieces) == 0 {
report.Outcome = OutcomeNoCodeChanged
return report, nil
}
newOut, newErr := runTest(ctx, r.root, opts.Test, opts.Timeout)
report.NewOutput = tail(newOut)
if ctx.Err() != nil {
return Report{}, ctx.Err()
}
if newErr != nil {
report.Outcome = OutcomeFailsOnNew
return report, nil
}
report.Outcome = OutcomeProven
for _, pc := range pieces {
res := PieceResult{File: pc.file, Line: pc.line, Lines: pc.lines, Snippet: pc.snippet}
if err := applyPiece(r, pc); err != nil {
if errors.Is(err, errStashTaken) {
return Report{}, err
}
if restoreErr := Restore(context.WithoutCancel(ctx), r.root); restoreErr != nil {
return Report{}, fmt.Errorf("undo piece %s:%d failed: %v; restore also failed: %w", pc.file, pc.line, err, restoreErr)
}
return Report{}, fmt.Errorf("undo piece %s:%d failed (files restored): %w", pc.file, pc.line, err)
}
out, testErr := runTest(ctx, r.root, opts.Test, opts.Timeout)
if err := Restore(context.WithoutCancel(ctx), r.root); err != nil {
return Report{}, fmt.Errorf("restore failed; recover with `test-check prove --restore`: %w", err)
}
if ctx.Err() != nil {
return Report{}, ctx.Err()
}
if testErr == nil {
res.Outcome = PieceUnguarded
report.Outcome = OutcomePieceNotTested
} else {
res.Outcome = PieceGuarded
res.BuildErrorSuspected = buildError.MatchString(out)
}
report.Pieces = append(report.Pieces, res)
}
return report, nil
}
Loading
Loading