Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ an example value. See `deploy/host/README.md`.

## [Unreleased]

### Added

- Once QA has finished, each failed lane in the QA step has a "Re-run" button, and the step has a "Re-run all failed" button. Both ask for confirmation, then run the lanes again on the same QA run; passed lanes keep their result.

## [0.2.8] - 2026-11-13

### Added
Expand Down
6 changes: 5 additions & 1 deletion backend/internal/api/release_routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type ReleaseService interface {
ApproveQA(ctx context.Context, email, id, head string) error
RetryQA(ctx context.Context, email, id, head string) error
SkipQA(ctx context.Context, email, id, head string) error
RerunQA(ctx context.Context, email, id, head, lane string) error
Recut(ctx context.Context, email, id, head string) error
Release(ctx context.Context, email, id, head string, confirmFailed bool) error
RetryRelease(ctx context.Context, email, id string) error
Expand Down Expand Up @@ -178,14 +179,15 @@ func mountRelease(r chi.Router, svc ReleaseService, origin func(http.Handler) ht
r.With(origin).Post("/api/release/releases/{id}/{action}", func(w http.ResponseWriter, req *http.Request) {
id, action := chi.URLParam(req, "id"), chi.URLParam(req, "action")
switch action {
case "approve-changelog", "retry-sync", "approve-qa", "retry-qa", "skip-qa", "recut", "release", "retry-release", "abandon":
case "approve-changelog", "retry-sync", "approve-qa", "retry-qa", "skip-qa", "rerun-qa", "recut", "release", "retry-release", "abandon":
default:
http.NotFound(w, req)
return
}
var body struct {
Head string `json:"head"`
ConfirmFailed bool `json:"confirm_failed"`
Lane string `json:"lane"` // rerun-qa: the lane, "" for every failed one
}
if !relBody(w, req, &body) {
return
Expand All @@ -203,6 +205,8 @@ func mountRelease(r chi.Router, svc ReleaseService, origin func(http.Handler) ht
err = svc.RetryQA(ctx, email, id, body.Head)
case "skip-qa":
err = svc.SkipQA(ctx, email, id, body.Head)
case "rerun-qa":
err = svc.RerunQA(ctx, email, id, body.Head, body.Lane)
case "recut":
err = svc.Recut(ctx, email, id, body.Head)
case "release":
Expand Down
4 changes: 4 additions & 0 deletions backend/internal/api/release_routes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ func (s *stubRelease) ApproveQA(ctx context.Context, email, id, head string) err
func (s *stubRelease) RetryQA(ctx context.Context, email, id, head string) error {
return s.rec("retry-qa", ctx)
}
func (s *stubRelease) RerunQA(ctx context.Context, email, id, head, lane string) error {
return s.rec("rerun-qa", ctx)
}
func (s *stubRelease) SkipQA(ctx context.Context, email, id, head string) error {
return s.rec("skip-qa", ctx)
}
Expand Down Expand Up @@ -118,6 +121,7 @@ var releaseRoutes = []struct{ method, path, body string }{
{"POST", "/api/release/releases/r1/approve-changelog", `{"head":"h"}`},
{"POST", "/api/release/releases/r1/retry-qa", `{"head":"h"}`},
{"POST", "/api/release/releases/r1/skip-qa", `{"head":"h"}`},
{"POST", "/api/release/releases/r1/rerun-qa", `{"head":"h","lane":"x"}`},
}

func TestReleaseRoutesRequireUser(t *testing.T) {
Expand Down
49 changes: 48 additions & 1 deletion backend/internal/release/actions.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ const (
auditRetrySync = "retry_sync"
auditApproveQA = "approve_qa"
auditRetryQA = "retry_qa"
auditSkipQA = "skip_qa" // Detail is the PR head the skip is for
auditSkipQA = "skip_qa" // Detail is the PR head the skip is for
auditRerunQA = "rerun_qa" // Detail is the lanes run again, one per line
auditRecut = "recut"
auditRetryRelease = "retry_release"
auditAbandon = "abandon"
Expand Down Expand Up @@ -473,6 +474,52 @@ func (s *Service) RetryQA(ctx context.Context, email, id, head string) error {
return nil
}

// RerunQA runs failed QA lanes again on the same QA run: the lane named lane,
// or every failed one when lane is "". GitHub runs a job again only once the
// whole run has completed, which is what QA "finished" means.
func (s *Service) RerunQA(ctx context.Context, email, id, head, lane string) error {
s.actMu.Lock()
defer s.actMu.Unlock()
// GitHub needs a moment to show the run as going again.
if err := s.retriedRecently(auditRerunQA, id); err != nil {
return err
}
r, err := s.guard(ctx, id, head, func(r Release) bool {
return r.Step == StepQA && r.PR != 0 && r.QA != nil && r.QA.State == "finished" && r.QA.runID != 0
})
if err != nil {
return err
}
var failed []string
var jobID int64
for _, l := range r.QA.Lanes {
if l.State != StateFailure {
continue
}
if lane == "" || l.Name == lane {
failed = append(failed, JobLanePrefix+l.Name)
jobID = l.jobID
}
}
switch {
case len(failed) == 0 && lane == "":
return fmt.Errorf("%w: no QA lane failed", ErrBadRequest)
case len(failed) == 0:
return fmt.Errorf("%w: %s is not a failed QA lane", ErrBadRequest, lane)
case lane == "":
err = s.gh.RerunFailedJobs(ctx, r.QA.runID)
default:
err = s.gh.RerunJob(ctx, jobID)
}
if err != nil {
return err
}
s.noteRetry(auditRerunQA, id)
s.record(email, auditRerunQA, r, strings.Join(failed, "\n"))
s.Invalidate()
return nil
}

// Recut rebases the release on the latest main and runs every workflow again:
// both labels come off and the cut is dispatched anew. It also retries a cut
// that failed.
Expand Down
106 changes: 105 additions & 1 deletion backend/internal/release/actions_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (fx *svcFixture) wantOneAudit(t *testing.T, action string) store.ReleaseAud

func (fx *svcFixture) wantNoWrites(t *testing.T) {
t.Helper()
for _, m := range []string{"AmendFile", "AddLabel", "RemoveLabel", "Comment", "Dispatch", "MergePull", "ClosePull", "DeleteBranch"} {
for _, m := range []string{"AmendFile", "AddLabel", "RemoveLabel", "Comment", "Dispatch", "RerunJob", "RerunFailedJobs", "MergePull", "ClosePull", "DeleteBranch"} {
if c := fx.gh.callsOf(m); len(c) != 0 {
t.Errorf("unexpected GitHub write: %v", c)
}
Expand Down Expand Up @@ -1038,3 +1038,107 @@ func TestSkipQADoesNotCoverANewHead(t *testing.T) {
t.Fatalf("err = %v", err)
}
}

func TestActionRerunQAOneLane(t *testing.T) {
ctx, id := context.Background(), "20261111T143210"
fx := newFixture(t)
fx.atQA(true, "completed", laneJob(200, "packages / pg_tde", "failure"), laneJob(201, "packages / pg_stat", "failure"), laneJob(202, "packages / pg_ok", "success"))
if err := fx.svc.RerunQA(ctx, actor, id, testHead, "packages / pg_tde"); err != nil {
t.Fatal(err)
}
if got := fx.gh.callsOf("RerunJob"); len(got) != 1 || got[0] != "RerunJob 200" {
t.Errorf("RerunJob = %v", got)
}
if n := len(fx.gh.callsOf("RerunFailedJobs")); n != 0 {
t.Errorf("RerunFailedJobs called %d times", n)
}
if a := fx.wantOneAudit(t, "rerun_qa"); a.Detail != JobLanePrefix+"packages / pg_tde" {
t.Errorf("audit = %+v", a)
}
// The run is going again: the lane is not failed any more, the others keep
// their result, and nothing can be run again until it finishes.
qa := fx.snapshot(t).Releases[0].QA
if qa.State != "running" || qa.Lanes[0].State != StateQueued || qa.Lanes[1].State != StateFailure || qa.Lanes[2].State != StateSuccess {
t.Fatalf("qa = %+v", qa)
}
fx.clock.advance(61 * time.Second)
if err := fx.svc.RerunQA(ctx, actor, id, testHead, "packages / pg_stat"); !errors.Is(err, ErrConflict) {
t.Fatalf("rerun while running: err = %v", err)
}
}

func TestActionRerunQAAllFailed(t *testing.T) {
ctx, id := context.Background(), "20261111T143210"
fx := newFixture(t)
fx.atQA(true, "completed", laneJob(200, "packages / pg_tde", "failure"), laneJob(201, "packages / pg_stat", "failure"), laneJob(202, "packages / pg_ok", "success"))
if err := fx.svc.RerunQA(ctx, actor, id, testHead, ""); err != nil {
t.Fatal(err)
}
if got := fx.gh.callsOf("RerunFailedJobs"); len(got) != 1 || !strings.HasPrefix(got[0], "RerunFailedJobs ") {
t.Errorf("RerunFailedJobs = %v", got)
}
if n := len(fx.gh.callsOf("RerunJob")); n != 0 {
t.Errorf("RerunJob called %d times", n)
}
want := JobLanePrefix + "packages / pg_tde\n" + JobLanePrefix + "packages / pg_stat"
if a := fx.wantOneAudit(t, "rerun_qa"); a.Detail != want {
t.Errorf("audit = %+v", a)
}
qa := fx.snapshot(t).Releases[0].QA
if qa.State != "running" || qa.Lanes[0].State != StateQueued || qa.Lanes[1].State != StateQueued || qa.Lanes[2].State != StateSuccess {
t.Fatalf("qa = %+v", qa)
}
}

func TestActionRerunQARefused(t *testing.T) {
ctx, id := context.Background(), "20261111T143210"
failedLane, passedLane := laneJob(200, "packages / pg_tde", "failure"), laneJob(201, "packages / pg_stat", "success")
cases := []struct {
name string
setup func(fx *svcFixture)
head string
lane string
want error
}{
{"QA still running", func(fx *svcFixture) {
fx.atQA(true, "in_progress", failedLane, Job{ID: 201, Name: JobLanePrefix + "packages / pg_stat", Status: "in_progress"})
}, testHead, "packages / pg_tde", ErrConflict},
{"QA not started", func(fx *svcFixture) { fx.atQA(false, "completed") }, testHead, "", ErrConflict},
{"stale head", func(fx *svcFixture) { fx.atQA(true, "completed", failedLane, passedLane) }, "oldhead", "", ErrConflict},
{"lane passed", func(fx *svcFixture) { fx.atQA(true, "completed", failedLane, passedLane) }, testHead, "packages / pg_stat", ErrBadRequest},
{"unknown lane", func(fx *svcFixture) { fx.atQA(true, "completed", failedLane, passedLane) }, testHead, "nope", ErrBadRequest},
{"nothing failed", func(fx *svcFixture) { fx.atQA(true, "completed", passedLane) }, testHead, "", ErrBadRequest},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
fx := newFixture(t)
c.setup(fx)
if err := fx.svc.RerunQA(ctx, actor, id, c.head, c.lane); !errors.Is(err, c.want) {
t.Fatalf("err = %v, want %v", err, c.want)
}
fx.wantNoWrites(t)
})
}
}

func TestActionRerunQAGitHubRefuses(t *testing.T) {
ctx, id := context.Background(), "20261111T143210"
fx := newFixture(t)
fx.atQA(true, "completed", laneJob(200, "packages / pg_tde", "failure"))
fx.gh.rerunErr = errors.New("boom")
if err := fx.svc.RerunQA(ctx, actor, id, testHead, ""); err == nil {
t.Fatal("no error")
}
if rows := fx.auditRows(t); len(rows) != 0 {
t.Errorf("audit rows: %+v", rows)
}
// A refused re-run starts no cooldown.
fx.gh.rerunErr = nil
if err := fx.svc.RerunQA(ctx, actor, id, testHead, ""); err != nil {
t.Fatal(err)
}
// An accepted one does, even if GitHub still showed the run as finished.
if err := fx.svc.RerunQA(ctx, actor, id, testHead, ""); !errors.Is(err, ErrConflict) {
t.Fatalf("second rerun: err = %v", err)
}
}
4 changes: 2 additions & 2 deletions backend/internal/release/derive.go
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ func QALanes(f Facts, units []UnitState, hasQALabel bool) QA {
if run == nil {
return qa
}
qa.URL = run.HTMLURL
qa.URL, qa.runID = run.HTMLURL, run.ID
cdnFailed := false
for _, j := range f.Jobs[run.ID] {
if isCDN(j) && jobState(j) == StateFailure {
Expand All @@ -297,7 +297,7 @@ func QALanes(f Facts, units []UnitState, hasQALabel bool) QA {
if !isLane(j) {
continue
}
l := Lane{Name: strings.TrimPrefix(j.Name, JobLanePrefix), URL: j.HTMLURL, State: jobState(j)}
l := Lane{Name: strings.TrimPrefix(j.Name, JobLanePrefix), URL: j.HTMLURL, State: jobState(j), jobID: j.ID}
for _, a := range f.Annotations[j.ID] {
if a.Title != AnnotationJenkins {
continue
Expand Down
49 changes: 49 additions & 0 deletions backend/internal/release/fake_github_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ type fakeGitHub struct {
closeErr error // returned by ClosePull only
createErr error // returned by CreatePull only
approveErr error // returned by ApproveRun only
rerunErr error // returned by RerunJob and RerunFailedJobs only
fileErrs map[string]error // "ref:path" -> error File returns for it
nextPR int
block chan struct{} // when non-nil, Branches waits for it to close
Expand Down Expand Up @@ -395,6 +396,54 @@ func (f *fakeGitHub) Dispatch(ctx context.Context, workflow string, inputs map[s
return err
}

// rerun puts run runID back in progress with the jobs pick chooses queued again,
// as GitHub does. f.mu must be held.
func (f *fakeGitHub) rerun(runID int64, pick func(Job) bool) {
for i := range f.runs {
if f.runs[i].ID == runID {
f.runs[i].Status, f.runs[i].Conclusion = "in_progress", ""
f.runs[i].Attempt++
}
}
for i, j := range f.jobs[runID] {
if pick(j) {
f.jobs[runID][i].Status, f.jobs[runID][i].Conclusion = "queued", ""
}
}
}

func (f *fakeGitHub) RerunJob(ctx context.Context, jobID int64) error {
err := f.record("RerunJob", jobID)
defer f.mu.Unlock()
if err == nil {
err = f.rerunErr
}
if err != nil {
return err
}
for runID, jobs := range f.jobs {
for _, j := range jobs {
if j.ID == jobID {
f.rerun(runID, func(x Job) bool { return x.ID == jobID })
}
}
}
return nil
}

func (f *fakeGitHub) RerunFailedJobs(ctx context.Context, runID int64) error {
err := f.record("RerunFailedJobs", runID)
defer f.mu.Unlock()
if err == nil {
err = f.rerunErr
}
if err != nil {
return err
}
f.rerun(runID, func(x Job) bool { return x.Conclusion == "failure" })
return nil
}

// ApproveRun starts a held run, as GitHub does.
func (f *fakeGitHub) ApproveRun(ctx context.Context, runID int64) error {
err := f.record("ApproveRun", runID)
Expand Down
17 changes: 17 additions & 0 deletions backend/internal/release/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ type GitHub interface {
AmendFile(ctx context.Context, branch, expectHead, path string, content []byte) (string, error)
Dispatch(ctx context.Context, workflow string, inputs map[string]string) error
ApproveRun(ctx context.Context, runID int64) error
// RerunJob runs one job of a completed run again, on the same run.
RerunJob(ctx context.Context, jobID int64) error
// RerunFailedJobs runs every failed job of a completed run again.
RerunFailedJobs(ctx context.Context, runID int64) error
WorkflowRuns(ctx context.Context, workflow string, limit int) ([]Run, error)
RunsForSHA(ctx context.Context, sha string) ([]Run, error)
Jobs(ctx context.Context, runID int64) ([]Job, error)
Expand Down Expand Up @@ -664,6 +668,19 @@ func (c *Client) ApproveRun(ctx context.Context, runID int64) error {
return err
}

// RerunJob runs one job again. GitHub accepts it only once the whole run has
// completed; the run keeps its id and gets a new attempt.
func (c *Client) RerunJob(ctx context.Context, jobID int64) error {
_, _, err := c.do(ctx, http.MethodPost, c.repoPath("actions/jobs/"+strconv.FormatInt(jobID, 10)+"/rerun"), nil, false)
return err
}

// RerunFailedJobs runs every failed job of a completed run again.
func (c *Client) RerunFailedJobs(ctx context.Context, runID int64) error {
_, _, err := c.do(ctx, http.MethodPost, c.repoPath("actions/runs/"+strconv.FormatInt(runID, 10)+"/rerun-failed-jobs"), nil, false)
return err
}

type ghRun struct {
ID int64 `json:"id"`
Name string `json:"name"`
Expand Down
20 changes: 20 additions & 0 deletions backend/internal/release/github_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -563,3 +563,23 @@ func TestClientComment(t *testing.T) {
t.Fatalf("body %v", body)
}
}

func TestClientRerun(t *testing.T) {
var got []string
c := newGHClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = append(got, r.Method+" "+r.URL.Path)
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{}`))
}))
ctx := context.Background()
if err := c.RerunJob(ctx, 200); err != nil {
t.Fatal(err)
}
if err := c.RerunFailedJobs(ctx, 7); err != nil {
t.Fatal(err)
}
want := "POST /repos/percona/obs-packaging/actions/jobs/200/rerun,POST /repos/percona/obs-packaging/actions/runs/7/rerun-failed-jobs"
if strings.Join(got, ",") != want {
t.Fatalf("requests = %v", got)
}
}
4 changes: 4 additions & 0 deletions backend/internal/release/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ type Lane struct {
JenkinsURL string `json:"jenkins_url"`
URL string `json:"url"`
State RunState `json:"state"`

jobID int64 // the GitHub job of the lane, to run it again
}

type QA struct {
Expand All @@ -80,6 +82,8 @@ type QA struct {
// The release may then be made whatever State is.
Skipped bool `json:"skipped"`
SkippedBy string `json:"skipped_by"` // who skipped; "" unless Skipped

runID int64 // the QA run, to run its failed jobs again; 0 while none is known
}

type Final struct {
Expand Down
Loading
Loading