Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 60 additions & 120 deletions .github/workflows/ci.yml

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion DESIGN.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# morca: design

November 1, 2026 · targets stable Mojo 1.2.0
November 1, 2026 · targets Mojo 1.3 (the nightly 1.3.0.dev2026111805)

This turns [VISION.md](VISION.md) into a buildable plan. It covers phase 1 in detail, sketches phases 2 to 6 and keyed partitions, and proposes answers to the vision's open questions. Everything marked *verified* was compiled and run on Mojo 1.2.0; every other claim about what Mojo accepts is checked in the PR that first needs it, and this file is corrected when one turns out wrong.

Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ lineage of datasets, and starts, resumes, reruns, backfills, cancels and
rebuilds through it.
## Install

morca is the mojoshelf tin `morca-mojo`, built for Mojo 1.2:
morca is the mojoshelf tin `morca-mojo`, built for the Mojo 1.3 nightly
(`mojo-compiler ==1.3.0.dev2026111805`, which a project pins too):

```sh
pixi init lake && cd lake
Expand Down
1 change: 1 addition & 0 deletions changes/550-mojo-1-3-nightly.minor.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **morca-mojo requires Mojo 1.3.0.dev2026111805:** it is built with that Mojo 1.3 nightly instead of 1.2.0, so a project pins `mojo-compiler = "==1.3.0.dev2026111805"` to add it. (#550)
4 changes: 2 additions & 2 deletions deploy/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
# pixi's -bookworm image, not its default (Ubuntu): the build stage then
# installs from Debian's mirrors, the same release as the runtime stage.
FROM ghcr.io/prefix-dev/pixi:0.78.0-bookworm AS build
ARG DISTA_REV=bc35e01ec26a9b29cdcdcb50d7757441ad99f0df
ARG K8S_REV=ab456806084467af45e9a50432c27aaade4f4338
ARG DISTA_REV=628025da9e280aa07f670c681d0acd5c5f4807f1
ARG K8S_REV=831f1ebde96d35351e1ea509f2514ecb858fc166
ARG PIPELINES="examples/toy.mojo examples/fanout.mojo"
RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev git ca-certificates \
&& rm -rf /var/lib/apt/lists/*
Expand Down
3,823 changes: 1,933 additions & 1,890 deletions deploy/runtime/pixi.lock

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions deploy/runtime/pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ platforms = ["linux-64", "linux-aarch64"]
preview = ["pixi-build"]

[dependencies]
mojo-compiler = "==1.2.0"
mojo-compiler = "==1.3.0.dev2026111805"
# The full mojo package, for the llvm-symbolizer the image ships. flare_mlake
# and json_mlake brought it in until they shipped as .mojoc packages.
mojo = "==1.2.0"
mojo = "==1.3.0.dev2026111805"
morca-mojo = { path = "../morca" }
2 changes: 1 addition & 1 deletion examples/hindcast/hindcast.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ def forecasts(mut ctx: Ctx, d: Day) raises -> Fetched:
# that isn't there yet is waited for; an old one is gone.
var msg = String(e)
var age = now() // 86400 - day_number(d)
if msg.find(NOT_PUBLISHED) >= 0 and age <= 2:
if NOT_PUBLISHED in msg and age <= 2:
raise defer(1800, msg)
raise e^

Expand Down
4 changes: 2 additions & 2 deletions examples/taxi/taxi.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ def load(mut ctx: Ctx, raw: RawFile) raises -> Loaded:

def _partition(path: String) -> String:
"""A data file's partition directory, `.../data/pickup_month=2024-01`."""
var at = path.rfind("/")
var at = path.rfind("/").or_else(-1)
return String(path[byte=0:at]) if at >= 0 else String()


Expand Down Expand Up @@ -160,7 +160,7 @@ def compact(mut ctx: Ctx, loads: Seq[Loaded]) raises -> Compacted:
var groups = List[List[FileScanTask]]()
for t in tasks:
var p = _partition(t.data_file.file_path)
if String(p[byte = p.rfind("/") + 1 :]) not in mine:
if String(p[byte = p.rfind("/").or_else(-1) + 1 :]) not in mine:
continue
var g = -1
for k in range(len(parts)):
Expand Down
4 changes: 2 additions & 2 deletions examples/taxi/tlc.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ def _is_parquet(io: FileIO, path: String) -> Bool:
var head = io.read_range(path, 0, 4)
var tail = io.read_range(path, n - 4, 4)
return (
String(StringSlice(unsafe_from_utf8=Span(head))) == "PAR1"
and String(StringSlice(unsafe_from_utf8=Span(tail))) == "PAR1"
String(StringSpan(unsafe_from_utf8=Span(head))) == "PAR1"
and String(StringSpan(unsafe_from_utf8=Span(tail))) == "PAR1"
)
except:
return False
Expand Down
4 changes: 2 additions & 2 deletions fuzz/harness.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def run_harness[
](name: StaticString, seeds: List[List[UInt8]], max_len: Int = 1024) raises:
var replay = getenv("FUZZ_REPLAY")
if replay:
var data = List[UInt8]()
var data: List[UInt8]
with open(replay, "r") as f:
data = f.read_bytes()
target(data)
Expand All @@ -54,7 +54,7 @@ def run_harness[
)


def bytes_of(s: StringSlice) -> List[UInt8]:
def bytes_of(s: StringSpan) -> List[UInt8]:
var out = List[UInt8]()
for b in s.as_bytes():
out.append(b)
Expand Down
23,834 changes: 4,583 additions & 19,251 deletions pixi.lock

Large diffs are not rendered by default.

190 changes: 82 additions & 108 deletions pixi.toml

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion scripts/mojo-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@
# moves it) and compiles only the functions an edit changed: a rebuild of
# build/morca takes about half the time. Its binaries run a few percent
# slower and are larger, so it is for development only: CI, the images and
# the releases build in the other environments, with `mojo`.
# the releases build in the other environments, with `mojo`. (pixi.toml has
# the environment off while no ouromo matches its Mojo.)
#
# Usage: scripts/mojo-build.sh <mojo build arguments...> -o <out>
set -eu
Expand Down
4 changes: 2 additions & 2 deletions scripts/pipelines.sh
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,13 @@ trap 'rm -f "$tmp"' EXIT INT TERM
echo
echo 'def main() raises:'
echo ' var me = String(argv()[0])'
echo ' var slash = me.rfind("/")'
echo ' var slash = me.rfind("/").or_else(-1)'
echo ' var name = String(me[byte = slash + 1 :]) if slash >= 0 else me'
echo ' var defines = String(me, ".defines")'
echo ' if exists(defines):'
echo ' with open(defines, "r") as f:'
echo ' for line in f.read().splitlines():'
echo ' var eq = line.find("=")'
echo ' var eq = line.find("=").or_else(-1)'
echo ' if eq < 0:'
echo ' continue'
echo ' var k = String(line[byte=:eq])'
Expand Down
2 changes: 1 addition & 1 deletion scripts/unit-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ build() {
echo
echo 'def main() raises:'
echo ' var me = String(argv()[0])'
echo ' var slash = me.rfind("/")'
echo ' var slash = me.rfind("/").or_else(-1)'
echo ' var name = String(me[byte = slash + 1 :]) if slash >= 0 else me'
kw=if
for l in $lines; do
Expand Down
44 changes: 27 additions & 17 deletions src/morca/_cli.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ from morca._internal.logs import (
from morca.tasks import workers
from morca.civil import Civil, now, parse_time
from morca._internal.crash import crash_summary
from morca.pipeline import Pipeline
from morca.pipeline import Pipeline, PipelineOf
from morca._internal.pool import pool_next_main, pool_wait_main
from morca._internal.sample import sample_main
from morca._internal.sqlite_store import SqliteStore, sqlite_path
Expand Down Expand Up @@ -156,12 +156,12 @@ def _command[P: type_of(Pipeline)](args: List[String]) raises -> Int:
_usage("set MORCA_DATABASE_URL or MORCA_STATE_URL, not both")
# A schedule is checked on every command, so a binary whose cron
# expression or zone is bad can't be registered or run.
comptime if P.scheduled():
_ = P.load_schedule()
comptime if PipelineOf[P].scheduled():
_ = PipelineOf[P].load_schedule()
if cmd == "fires":
return _fires[P](args)
if cmd == "describe" and len(args) == 2:
print(P.describe())
print(PipelineOf[P].describe())
return 0
if cmd == "keep-log" and len(args) == 5:
# For the ec2 runner's head instance: the driver's output, appended
Expand Down Expand Up @@ -230,7 +230,7 @@ def _command[P: type_of(Pipeline)](args: List[String]) raises -> Int:
return 0
if cmd == "sense" and len(args) == 2:
var wh = getenv("MORCA_WAREHOUSE")
var names = P.sensed_datasets()
var names = PipelineOf[P].sensed_datasets()
if len(names) > 0 and not wh:
raise Error("morca: sense needs MORCA_WAREHOUSE")
for d in names:
Expand Down Expand Up @@ -337,12 +337,12 @@ def _run[
) raises -> Int:
"""`run`: records the pipeline and a new run of `keys` in `store`, and
runs it."""
var hash = hex(P.registry_hash())
var hash = hex(PipelineOf[P].registry_hash())
store.migrate()
# Only `morca register` moves a scheduled pipeline's code.
var current = store.scheduled_hash(P.name)
if not current:
store.register(P.name, hash, P.describe())
store.register(P.name, hash, PipelineOf[P].describe())
elif current != hash and not new_code:
print(
"morca: pipeline ",
Expand All @@ -363,7 +363,10 @@ def _run[
key.write("," if j > 0 else "", keys[j])
var run = store.create_run(P.name, hash, key, "manual", owner)
store.ran_version(
run, store.version(P.name, hash, P.describe(), getenv("MORCA_IMAGE"))
run,
store.version(
P.name, hash, PipelineOf[P].describe(), getenv("MORCA_IMAGE")
),
)
print("run", run)
return _execute[P](rt, store, run, owner, keys, RunResult())
Expand All @@ -386,7 +389,7 @@ def _resume[
", not ",
P.name,
)
var hash = hex(P.registry_hash())
var hash = hex(PipelineOf[P].registry_hash())
if info.registry_hash != hash and not allow_new_code:
raise Error(
"morca: run ",
Expand All @@ -400,7 +403,10 @@ def _resume[
var owner = process_owner()
store.claim(run, owner, _stale_seconds())
store.ran_version(
run, store.version(P.name, hash, P.describe(), getenv("MORCA_IMAGE"))
run,
store.version(
P.name, hash, PipelineOf[P].describe(), getenv("MORCA_IMAGE")
),
)
var interrupted = store.interrupt(run)
var prior = store.prior(run)
Expand Down Expand Up @@ -472,7 +478,7 @@ def _status[S: RunStore](mut store: S, args: List[String]) raises -> Int:
def _fires[P: type_of(Pipeline)](args: List[String]) raises -> Int:
"""The schedule's next fires after now (or `--after`), each with the
partition it runs: UTC time, wall-clock time, partition."""
comptime if not P.scheduled():
comptime if not PipelineOf[P].scheduled():
_usage(String("pipeline ", P.name, " has no schedule"))
return 2
else:
Expand All @@ -496,20 +502,20 @@ def _print_fires[P: type_of(Pipeline)](args: List[String]) raises -> Int:
except:
_usage(String("not a count: ", args[i]))
i += 1
var s = P.load_schedule()
var s = PipelineOf[P].load_schedule()
var t = after
for _ in range(count):
t = s.next_fire(t)
print(
String(Civil.of(t), "Z"),
String(Civil.of(s.zone.to_local(t))),
P.fills_for(t, s),
PipelineOf[P].fills_for(t, s),
)
return 0


def _parent(path: String) -> String:
var at = path.rfind("/")
var at = path.rfind("/").or_else(-1)
return String(path[byte=:at]) if at > 0 else String(".")


Expand Down Expand Up @@ -540,11 +546,15 @@ def _execute[
the run over), the run is left `running`; once its heartbeat is stale it
can be resumed.
"""
store.own(run, owner, hex(P.registry_hash()))
store.start_heartbeat(heartbeat_interval(_stale_seconds()), P.timeouts())
store.own(run, owner, hex(PipelineOf[P].registry_hash()))
store.start_heartbeat(
heartbeat_interval(_stale_seconds()), PipelineOf[P].timeouts()
)
var res: RunResult
try:
res = P.execute(store, prior^, keys, run, Int(Pointer(to=rt)))
res = PipelineOf[P].execute(
store, prior^, keys, run, Int(Pointer(to=rt))
)
except e:
store.stop_heartbeat()
if not store.cancel_requested():
Expand Down
2 changes: 1 addition & 1 deletion src/morca/_internal/alerts.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ from morca._internal.watchdog import active_trips

def _first_line(text: String, limit: Int = 200) -> String:
var line = String(text.strip())
var nl = line.find("\n")
var nl = line.find("\n").or_else(-1)
if nl >= 0:
var first = String(line[byte=:nl])
line = first^
Expand Down
2 changes: 1 addition & 1 deletion src/morca/_internal/api/actions.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ def bulk(mut db: State, call: Call) raises -> Page:
)
if what == "backfill":
to.write("&queued=", b.queued)
return Page(303, j^, String(), location=to^)
return Page(303, j^, String(), location=to)


def _verb(what: String) -> String:
Expand Down
2 changes: 1 addition & 1 deletion src/morca/_internal/api/worker.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ def answer(url: String, path: String, token: String, body: String) -> Page:
def worker_response(url: String, req: Request) -> Response:
"""The response to a worker's `req`, which the UI's server and the
worker listener both send."""
var q = req.url.find("?")
var q = req.url.find("?").or_else(-1)
var path = String(req.url[byte=:q]) if q >= 0 else req.url
var page = answer(
url,
Expand Down
10 changes: 5 additions & 5 deletions src/morca/_internal/aws.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ def _q(s: String) -> String:
def env_problem(kv: String) -> String:
"""Why `kv` can't be one of the control instance's settings
(`--env KEY=VALUE`, the stack's `ControlEnv`), or empty if it can."""
var eq = kv.find("=")
var eq = kv.find("=").or_else(-1)
if eq <= 0:
return String("is KEY=VALUE, not ", kv)
var key = String(kv[byte=:eq])
Expand Down Expand Up @@ -347,10 +347,10 @@ def lock_morca_version(lock: String) -> String:
if line.startswith("- "):
in_morca = line.startswith("- conda_source: morca-mojo[")
var name = "/morca-mojo-"
var at = line.find(name)
var at = line.find(name).or_else(-1)
if line.startswith("- conda: ") and at >= 0:
var rest = String(line[byte = at + name.byte_length() :])
var dash = rest.find("-")
var dash = rest.find("-").or_else(-1)
if dash > 0:
return String(rest[byte=:dash])
continue
Expand Down Expand Up @@ -515,7 +515,7 @@ def _push(o: _Opts) raises -> Int:
)
return 2
var repo = _output(_outputs(o), "Repository")
var registry = String(repo[byte = : repo.find("/")])
var registry = String(repo[byte = : repo.find("/").or_else(-1)])
var tag = _tag(o)
var ctx = String(_DIR, "/context")
_run(
Expand Down Expand Up @@ -703,7 +703,7 @@ def _register(o: _Opts) raises -> Int:
var text: String
with open(config_path, "r") as f:
text = f.read()
var slash = config_path.rfind("/")
var slash = config_path.rfind("/").or_else(-1)
reg.write(
" --config-b64 ",
_q(b64encode(text)),
Expand Down
8 changes: 4 additions & 4 deletions src/morca/_internal/aws_replace.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ struct _State(Movable):
with open(self.path, "r") as f:
text = f.read()
for line in text.split("\n"):
var eq = line.find("=")
var eq = line.find("=").or_else(-1)
if eq > 0:
self.keys.append(String(line[byte=:eq]))
self.values.append(String(line[byte = eq + 1 :]))
Expand Down Expand Up @@ -485,7 +485,7 @@ def _check(prefix: String, smoke: String) -> List[String]:
),
]
if smoke:
var eq = smoke.find("=")
var eq = smoke.find("=").or_else(-1)
var pipeline = String(smoke[byte=:eq])
var key = String(smoke[byte = eq + 1 :])
var last = String(
Expand Down Expand Up @@ -567,7 +567,7 @@ def _deploy(
o: _Opts, st: _State, step: String, var params: List[String]
) raises:
for p in slot_pins(o, _outputs(o)):
var key = String(p[byte = : p.find("=") + 1])
var key = String(p[byte = : p.find("=").or_else(-1) + 1])
var given = False
for q in params:
if q.startswith(key):
Expand Down Expand Up @@ -1063,7 +1063,7 @@ def replace_control(args: List[String]) raises -> Int:
return 2
_ = _seconds(_keep(o))
if o.smoke:
var eq = o.smoke.find("=")
var eq = o.smoke.find("=").or_else(-1)
if eq <= 0 or eq + 1 >= o.smoke.byte_length():
print("morca aws replace-control: --smoke is PIPELINE=KEY")
return 2
Expand Down
Loading
Loading