From 667d4a4b383c890cfaad7ea61c60acf2b049cbed Mon Sep 17 00:00:00 2001 From: Marius S <39998+winding-lines@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:58:55 -0800 Subject: [PATCH 1/2] Resume, trigger cli, until unpaused, generated vocabulary tables (#378) - UI: Retry and Retry failed are Resume and Resume failed; POST /runs/bulk takes do=resume, and do=retry (and an old did=retry link) as its alias. - morca start records trigger cli (an API token, a JSON action, or the in-process command); the site's Start one run stays ui. cli is in the runs filter and has help; the backfill and ui help say what sets them. - paused: "until unpaused". - docs/ui.md's run-status, step-status and trigger tables are generated from morca._internal.ui.terms (pixi run docs-vocabulary), and test-terms fails when they differ. The nonexistent "retrying" status is gone from the cancel help and docs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017TycszSbyw92Efu4ZeCYL6 --- README.md | 2 +- docs/ui.md | 48 ++++++++++++-- pixi.toml | 6 +- scripts/unit-tests.sh | 1 + src/morca/_internal/actions.mojo | 13 ++-- src/morca/_internal/api/actions.mojo | 33 ++++++---- src/morca/_internal/command.mojo | 5 +- src/morca/_internal/db/actions.mojo | 19 ++++-- src/morca/_internal/ui/parts.mojo | 10 +-- src/morca/_internal/ui/runs.mojo | 42 +++--------- src/morca/_internal/ui/terms.mojo | 85 ++++++++++++++++++++++++- src/morca/_internal/views/commands.mojo | 7 +- src/morca/_internal/web/router.mojo | 3 +- src/morca/_internal/web/server.mojo | 3 + tests/remote.sh | 11 +++- tests/test_terms.mojo | 62 ++++++++++++++++++ tests/ui.sh | 25 +++++--- 17 files changed, 291 insertions(+), 84 deletions(-) create mode 100644 tests/test_terms.mojo diff --git a/README.md b/README.md index d8a40254..c505a867 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ pixi run -e data taxi && pixi run -e data ./build/taxi run --partitions 2023-01. pixi run morca && ./build/morca register ./build/toy && ./build/morca scheduler # needs MORCA_DATABASE_URL ./build/morca register ./build/taxi && ./build/morca backfill taxi 2011-01..2024-12 # 19 GB of TLC files ./build/morca backfill --rerun --allow-append taxi 2024-03 # queue a month again, though it succeeded; taxi's load appends, so say so -./build/morca cancel taxi @all # cancel its queued, running and retrying runs (or KEYS) +./build/morca cancel taxi @all # cancel its live runs, and failed ones due a retry (or KEYS) ./build/morca pause taxi # the scheduler starts none of its runs until morca unpause taxi ./build/morca retire taxi # no longer registered, its runs and tables kept; morca register brings it back ./build/morca watchdog status # the watchdog pauses pipelines on a launch storm, runaway cost, idle workers, diff --git a/docs/ui.md b/docs/ui.md index c5210d8b..10c58004 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -76,7 +76,7 @@ refreshing with them applied (its `/runs/table` fragment carries them). | Parameter | | |---|---| | `?status=` | runs with any of the statuses: `?status=failed&status=queued`, or `failed,queued`. The box lists every status with its runs under the other filters, which the JSON has as `counts` | -| `?trigger=` | `schedule`, `sensor`, `backfill`, `fill`, `manual`, `ui`, `rerun`, `rebuild` | +| `?trigger=` | `schedule`, `sensor`, `backfill`, `fill`, `manual`, `ui`, `cli`, `rerun`, `rebuild` (each in [the table below](#help-on-every-page)) | | `?pipeline=` | on `/runs` | | `?from=`, `?to=` | created on or after, and by the end of, a day (`2024-06-01`) or at a time (`2024-06-01T09:30`), UTC; one that isn't a time is left out | | `?since=` | created in the last `1h`, `24h`, `7d` (any `h` or `d`), or `today` since midnight UTC: the quick picks beside the dates | @@ -96,8 +96,8 @@ and a few with one. Logged in (or with login off), each row has a checkbox and the bar over the table acts on the runs ticked: **Cancel** (queued, deferred and -running ones, as each row's Cancel does; the others are left), **Retry -failed** (failed and stalled ones queued again, as each row's Retry does, +running ones, as each row's Cancel does; the others are left), **Resume +failed** (failed and stalled ones queued again, as each row's Resume does, except one whose code has changed, which is resumed from its own page) and **Backfill again** (what `morca backfill --rerun` does, over the partitions of the runs ticked, a backfill per pipeline; one whose steps @@ -108,7 +108,8 @@ there are still that many, and if not says how many there are now, with a button to do it to those. One action does at most 2,000 runs. Logged out, the boxes and buttons give way to a link to log in. -It's a `POST /runs/bulk` with `do` (`cancel`, `retry`, `backfill`), each +It's a `POST /runs/bulk` with `do` (`cancel`, `resume`, `backfill`; `retry`, +its name before 0.35, is taken for `resume`), each run ticked as `pick=`, or `all=1&expect=` and the list's filters (`view`, `pipeline`, `status`, ...). It answers `303` back to the list, which says what it did ("Cancelled 12 runs, 3 already finished"), or to @@ -488,6 +489,45 @@ Configuration and Actions tables), a value's by its kind and value, and a setting's by its key. The UI adds a header's "?" to every `` it has a text for as it answers, so a new table gets them by using the same words. +The statuses and triggers, as the UI explains them (generated from that +module: `pixi run docs-vocabulary` writes them, and `pixi run test-terms` +fails when they differ). A run started with `morca start` before 0.35 has +trigger `ui`, as one from the site does. + + + +| Run status | Means | +|---|---| +| `queued` | Waiting to start: its turn, or room for its placement. | +| `running` | Going now. | +| `deferred` | A step asked to run again later; the run waits until then. | +| `succeeded` | Done, every step succeeded. | +| `failed` | Something failed after its last retry. Resume runs what hadn't succeeded. | +| `cancelled` | Stopped on request, before its next attempt. | + +| Step status | Means | +|---|---| +| `pending` | Not started yet in this run. | +| `running` | Going now. | +| `deferred` | A step asked to run again later; the run waits until then. | +| `succeeded` | Done, every step succeeded. | +| `failed` | Something failed after its last retry. Resume runs what hadn't succeeded. | +| `skipped` | Not run: a step it waits for didn't succeed. | + +| Trigger | Means | +|---|---| +| `schedule` | Its schedule fired. | +| `sensor` | Its sensor saw a new snapshot of a table it reads. | +| `backfill` | morca backfill, or a Backfill button here: Start runs' or a runs list's Backfill again. | +| `manual` | A binary's run command started it, outside the scheduler. | +| `ui` | Start runs' Start one run over the whole range, on this site. | +| `cli` | morca start: one run over the partitions it was given. | +| `rerun` | A rerun of an ended run, from a step on. | +| `rebuild` | A rebuild of everything downstream of a dataset. | +| `fill` | The fill: the scheduler working through the range by itself. | + + + The Configuration table (a pipeline's Overview) has each key's "?", and its From column links to where the value is set, where morca knows it: diff --git a/pixi.toml b/pixi.toml index 4328573e..5f7acbc4 100644 --- a/pixi.toml +++ b/pixi.toml @@ -283,6 +283,10 @@ test-defer = { cmd = "scripts/with-postgres.sh tests/defer.sh" } test-sample = { cmd = "scripts/unit-tests.sh build executor-core && scripts/unit-tests.sh run test-sample && tests/sample.sh" } # A run's machines on its page (morca.metrics): peaks, warnings and charts. test-metrics = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-metrics" } +# The UI's vocabulary (morca._internal.ui.terms) and docs/ui.md's tables of it; +# docs-vocabulary writes the tables again from it. +test-terms = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-terms" } +docs-vocabulary = { cmd = "scripts/unit-tests.sh build scheduler-ui && MORCA_WRITE_DOCS=1 scripts/unit-tests.sh run test-terms" } # Needs docker, kind and kubectl; builds the image of deploy/Dockerfile. test-kind = { cmd = "tests/kind.sh" } # The binaries most tests share, built at once before the tests that copy @@ -295,7 +299,7 @@ test-executor-core = { depends-on = ["build-shared", "test-sample", "test-steps" test-executor-workers = { depends-on = ["build-shared", "test-ec2", "test-autoscale", "test-no-workers", "test-pool", "test-hcloud", "test-pool-scale", "test-pool-scale-ec2", "test-pool-image", "test-run-cluster", "test-secrets", "test-placement", "test-grants", "test-pressure", "test-lost-workers"] } test-executor = { depends-on = ["test-executor-core", "test-executor-workers"] } # The scheduler, the UI, and what a registration and its code control: -test-scheduler-ui = { depends-on = ["build-shared", "test-toml", "test-upgrade", "test-config", "test-fill", "test-fill-version", "test-fill-requests", "test-priority", "test-pool-fills", "test-run-locks", "test-metrics", "test-scheduler", "test-ui", "test-fleet", "test-system", "test-actions", "test-calendar", "test-ui-login", "test-ui-drain", "test-remote", "test-worker-api", "test-pause", "test-older-binary", "test-watchdog", "test-notify", "test-versions", "test-manual-run", "test-new-code", "test-defer", "test-dead-runs", "test-adopt", "test-overlap", "test-outdated", "test-concurrency", "test-one-run-per-key", "test-hourly", "test-weekly", "test-sense-errors", "test-stop", "test-binary-timeouts"] } +test-scheduler-ui = { depends-on = ["build-shared", "test-toml", "test-upgrade", "test-config", "test-fill", "test-fill-version", "test-fill-requests", "test-priority", "test-pool-fills", "test-run-locks", "test-metrics", "test-terms", "test-scheduler", "test-ui", "test-fleet", "test-system", "test-actions", "test-calendar", "test-ui-login", "test-ui-drain", "test-remote", "test-worker-api", "test-pause", "test-older-binary", "test-watchdog", "test-notify", "test-versions", "test-manual-run", "test-new-code", "test-defer", "test-dead-runs", "test-adopt", "test-overlap", "test-outdated", "test-concurrency", "test-one-run-per-key", "test-hourly", "test-weekly", "test-sense-errors", "test-stop", "test-binary-timeouts"] } test = { depends-on = ["test-executor", "test-scheduler-ui"] } toy = { cmd = "scripts/mojo-build.sh -I src $DEP_INC examples/toy.mojo -o build/toy" } fanout = { cmd = "scripts/mojo-build.sh -I src $DEP_INC examples/fanout.mojo -o build/fanout" } diff --git a/scripts/unit-tests.sh b/scripts/unit-tests.sh index 0e6c233e..cd4fd79f 100755 --- a/scripts/unit-tests.sh +++ b/scripts/unit-tests.sh @@ -81,6 +81,7 @@ scheduler-ui:tests/test_upgrade.mojo:test-upgrade scheduler-ui:tests/test_config.mojo:test-config scheduler-ui:tests/test_fill.mojo:test-fill scheduler-ui:tests/test_metrics.mojo:test-metrics +scheduler-ui:tests/test_terms.mojo:test-terms scheduler-ui:tests/test_remote.mojo:test-remote scheduler-ui:tests/test_worker.mojo:test-worker-api scheduler-ui:tests/test_watchdog.mojo:test-watchdog diff --git a/src/morca/_internal/actions.mojo b/src/morca/_internal/actions.mojo index 794a6ed2..d5c23cfc 100644 --- a/src/morca/_internal/actions.mojo +++ b/src/morca/_internal/actions.mojo @@ -78,8 +78,11 @@ def _pipeline(mut db: State, name: String) raises -> Optional[Registered]: return ps[0].copy() -def start_run(mut db: State, pipeline: String, keys: String) raises -> Outcome: - """A queued run over the partitions in `keys`, trigger `ui`. Refused +def start_run( + mut db: State, pipeline: String, keys: String, trigger: String = "ui" +) raises -> Outcome: + """A queued run over the partitions in `keys`, with `trigger`: `ui` from + the Start runs form, `cli` from `morca start`. Refused while a live run has one of them, which a second run would run beside; a partition that's done is run again.""" var p = _pipeline(db, pipeline) @@ -111,8 +114,8 @@ def start_run(mut db: State, pipeline: String, keys: String) raises -> Outcome: " cancel it", ) ) - var id = actions_db.add_ui_run( - tx, pipeline, p.value().registry_hash, joined + var id = actions_db.add_started_run( + tx, pipeline, p.value().registry_hash, joined, trigger ) tx.commit() return Outcome.done(Int(id)) @@ -395,7 +398,7 @@ def cancel_runs_picked(mut db: State, runs: List[Int]) raises -> Bulk: def resume_failed(mut db: State, runs: List[Int]) raises -> Bulk: - """Queues again, as `resume_run` does each (Retry on a runs list), those + """Queues again, as `resume_run` does each (Resume on a runs list), those of `runs` that failed or stalled. One whose pipeline's code has changed since is refused, to be resumed from its page; the others are skipped.""" var b = Bulk.none() diff --git a/src/morca/_internal/api/actions.mojo b/src/morca/_internal/api/actions.mojo index 7d0ee86b..fa108439 100644 --- a/src/morca/_internal/api/actions.mojo +++ b/src/morca/_internal/api/actions.mojo @@ -5,14 +5,14 @@ that changes something, through `morca._internal.actions`. |---|---| | `/pipelines//runs` | start a run over `partitions` | | `/pipelines//backfill` | a run per partition in `partitions` that has none; `rerun=1` for done ones too | -| `/pipelines//cancel` | cancel its queued, running and retrying runs over `partitions` (`@all` or empty: all) | +| `/pipelines//cancel` | cancel its queued, deferred and running runs, and failed ones due a retry, over `partitions` (`@all` or empty: all) | | `/pipelines//pause`, `/unpause` | stop the scheduler starting its runs, or let it again | | `/watchdog//snooze` | silence a watchdog rule for `duration` (`2h`, `30m`, `1d`; `off` ends it) | | `/watchdog//ack`, `/watchdog/all/ack` | acknowledge a watchdog trip, or all of them | | `/runs//resume` | queue a failed, cancelled or stalled run again; `allow_new_code=1` confirms new code | | `/runs//rerun` | a new run from `step` on, the steps before it copied | | `/runs//cancel` | cancel a run, before its next attempt if it's running | -| `/runs/bulk` | `do=cancel`, `retry` or `backfill` the runs picked from a list: each `pick=`, or with `all=1` every run its filters (`view`, `pipeline`, `status`, ...) match, if there are still `expect` of them | +| `/runs/bulk` | `do=cancel`, `resume` (or its old name `retry`) or `backfill` the runs picked from a list: each `pick=`, or with `all=1` every run its filters (`view`, `pipeline`, `status`, ...) match, if there are still `expect` of them | | `/datasets//rebuild` | rebuild every pipeline downstream over `partitions` | Each answers `303 See Other` to the page of what it made (a script gets @@ -134,11 +134,17 @@ comptime _FROM_FILE = ( def start(mut db: State, call: Call) raises -> Page: - """`POST /pipelines//runs`: one run over `partitions`.""" + """`POST /pipelines//runs`: one run over `partitions`, trigger + `cli` from the `morca` command (`call.source`), else `ui`.""" var keys = call.field("partitions") if _from_file(keys): return bad_request(_FROM_FILE) - var o = start_run(db, call.param("name"), keys) + var o = start_run( + db, + call.param("name"), + keys, + "cli" if call.source == "cli" else "ui", + ) return _acted( o, _operate_href(call), @@ -336,7 +342,8 @@ def _ids_json(mut j: String, ids: List[Int]): def bulk(mut db: State, call: Call) raises -> Page: - """`POST /runs/bulk`: `do` (`cancel`, `retry` or `backfill`) to the runs + """`POST /runs/bulk`: `do` (`cancel`, `resume` or `backfill`; `retry` is + `resume`'s old name) to the runs picked from a list (`morca._internal.ui.runs`), each a `pick` field, or with `all=1` all those the list's filters match (the form's `view`, `pipeline`, `status`, `trigger`, `from`, `to`, `since`, `key`, `run`), @@ -353,8 +360,10 @@ def bulk(mut db: State, call: Call) raises -> Page: q.view = "list" var back = q.href() var what = call.field("do") - if what != "cancel" and what != "retry" and what != "backfill": - return bad_request("do is cancel, retry or backfill") + if what == "retry": + what = "resume" # its name before #378, still taken + if what != "cancel" and what != "resume" and what != "backfill": + return bad_request("do is cancel, resume or backfill") var ids = List[Int]() if call.field("all") == "1": var n = count_runs(db, q) @@ -416,7 +425,7 @@ def bulk(mut db: State, call: Call) raises -> Page: var b: Bulk if what == "cancel": b = cancel_runs_picked(db, ids) - elif what == "retry": + elif what == "resume": b = resume_failed(db, ids) else: b = backfill_picked(db, ids, call.field("allow_append") == "1") @@ -438,7 +447,9 @@ def bulk(mut db: State, call: Call) raises -> Page: "Backfill them, appending again", ) if call.user: - var action = String("/resume") if what == "retry" else String("/cancel") + var action = String("/resume") if what == "resume" else String( + "/cancel" + ) if what != "backfill": for id in b.done: audit( @@ -494,6 +505,6 @@ def _verb(what: String) -> String: """A bulk action's button, before "all ".""" if what == "cancel": return String("Cancel") - if what == "retry": - return String("Retry the failed of") + if what == "resume": + return String("Resume the failed of") return String("Backfill again") diff --git a/src/morca/_internal/command.mojo b/src/morca/_internal/command.mojo index a4079ac0..5a25ad96 100644 --- a/src/morca/_internal/command.mojo +++ b/src/morca/_internal/command.mojo @@ -15,7 +15,7 @@ Its commands: ./build/morca config export # a morca.toml from the registrations ./build/morca backfill taxi 2011-01..2024-12 ./build/morca backfill --rerun taxi 2024-03 # again, though it succeeded - ./build/morca cancel taxi @all # its queued, running and retrying runs + ./build/morca cancel taxi @all # its live runs, and failed ones due a retry ./build/morca pause taxi # start none of its runs; morca unpause taxi ./build/morca tick ./build/morca scheduler @@ -160,7 +160,8 @@ comptime USAGE = """usage: morca backfill --cancel (its fill request, and its runs) morca start KEY[..KEY][,...] (one run over them all) morca cancel KEY[..KEY][,...]|@all - (its queued, running and retrying runs over those keys) + (its queued, deferred and running runs over those keys, + and failed ones due a retry) morca cancel morca resume [--allow-new-code] morca rerun [--allow-append] diff --git a/src/morca/_internal/db/actions.mojo b/src/morca/_internal/db/actions.mojo index 1015dd3c..ad388b4d 100644 --- a/src/morca/_internal/db/actions.mojo +++ b/src/morca/_internal/db/actions.mojo @@ -23,18 +23,27 @@ def _cancel_set(kill_seconds: StaticString) -> String: ) -def add_ui_run( - mut tx: Transaction, pipeline: String, registry_hash: String, keys: String +def add_started_run( + mut tx: Transaction, + pipeline: String, + registry_hash: String, + keys: String, + trigger: String, ) raises -> Int64: - """A new queued run of `pipeline` over `keys`, trigger `ui`; its id.""" + """A new queued run of `pipeline` over `keys`, with `trigger` (`ui` or + `cli`); its id.""" return ( tx.query( ( "INSERT INTO orca.runs (pipeline, registry_hash, partitions," - " trigger, status) VALUES ($1, $2, $3, 'ui', 'queued')" + " trigger, status) VALUES ($1, $2, $3, $4, 'queued')" " RETURNING id" ), - Params().text(pipeline).text(registry_hash).text(keys), + Params() + .text(pipeline) + .text(registry_hash) + .text(keys) + .text(trigger), ) .row(0) .int64("id") diff --git a/src/morca/_internal/ui/parts.mojo b/src/morca/_internal/ui/parts.mojo index 16cad21c..e1ff198a 100644 --- a/src/morca/_internal/ui/parts.mojo +++ b/src/morca/_internal/ui/parts.mojo @@ -12,7 +12,7 @@ from morca._internal.db.run_rows import Usage from morca._internal.ui.layout import act_button, fact_item from morca._internal.usage import bytes_text, cpu_text, usage_text from morca._internal.views.code import CodeLinks, CodeRef, RunCode -from morca._internal.views.commands import cancel_cli, retry_cli, shell_quote +from morca._internal.views.commands import cancel_cli, resume_cli, shell_quote from morca._internal.views.runs import RunResources, RunRow from morca._internal.db.run_view import VersionRow @@ -289,7 +289,7 @@ def cancel_tell(r: RunRow) -> String: def row_actions(mut h: Html, r: RunRow): - """A runs table row's Cancel (queued or running) or Retry (failed).""" + """A runs table row's Cancel (queued or running) or Resume (failed).""" if (r.status == "queued" or r.status == "running") and not ( r.cancel_requested ): @@ -309,9 +309,9 @@ def row_actions(mut h: Html, r: RunRow): act_button( h, String("/runs/", r.id, "/resume"), - "Retry", - "retry", - retry_cli(r), + "Resume", + "resume", + resume_cli(r), ) diff --git a/src/morca/_internal/ui/runs.mojo b/src/morca/_internal/ui/runs.mojo index fe9f3486..4dfa96ce 100644 --- a/src/morca/_internal/ui/runs.mojo +++ b/src/morca/_internal/ui/runs.mojo @@ -1,7 +1,7 @@ """`/runs`, the runs list: newest first, filtered by pipeline, status (any of several, each with its count), trigger, when it was created, a run and its partitions, a page at a time (`morca._internal.views.runs.RunsPage`), and its -runs picked to cancel, retry or backfill again in one go +runs picked to cancel, resume or backfill again in one go (`morca._internal.api.actions.bulk`); `/runs/table` is a list's page as the fragment it refreshes from, for `/runs`, a pipeline's Runs tab and the home page. """ @@ -26,6 +26,7 @@ from morca._internal.ui.layout import ( ) from morca._internal.ui.parts import runs_table_html from morca._internal.ui.recent import recent_table +from morca._internal.ui.terms import RUN_STATUSES, RUN_TRIGGERS from morca._internal.views.code import runs_code from morca._internal.views.runs import ( BULK_MAX, @@ -39,33 +40,6 @@ from morca._internal.views.runs import ( from morca._internal.web.page import Page -comptime RUN_STATUSES: List[String] = [ - "queued", - "running", - "deferred", - "succeeded", - "failed", - "cancelled", -] -"""The statuses a run takes (`orca.runs.status`), in the order a run goes -through them; a deferred run waits for its `retry_at`.""" - - -comptime RUN_TRIGGERS: List[String] = [ - "schedule", - "sensor", - "backfill", - "manual", - "ui", - "rerun", - "rebuild", - "fill", -] -"""The triggers morca writes (`orca.runs.trigger`): the scheduler's fires and -sensors, `backfill`, a binary's `run`, the UI's start button, a rerun, a -rebuild and a fill (`morca._internal.fill`). The filter adds any others the table has.""" - - comptime _SUBMIT_ON_CHANGE = ( "