Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
## 0.1.0 - unreleased

- Hindcast as a project of its own, from morca's examples/hindcast, with its image and CI.
- On morca 0.31.0, with its Data tab.
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ crate-type = ["cdylib", "rlib"]
[dependencies]
# The morca the pipeline pins (pixi.toml's morca-mojo): the ABI version is
# checked when the step runs. Move the two together.
morca-worker = { git = "https://github.com/morca-dev/morca", tag = "v0.27.0" }
morca-worker = { git = "https://github.com/morca-dev/morca", tag = "v0.31.0" }
# Pure Rust only, so the library links nothing but libc and fits the
# distroless runtime image: no openjpeg or libaec (GFS 2 m temperature is
# template 5.3, which grib decodes itself; these features cover the rest).
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile.morca
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
# The stages that RUN are Debian (bookworm, the runtime's glibc); the final
# stage is the runtime's distroless base, which has only busybox `sh`, so it
# only copies files in.
ARG MORCA_VERSION=0.27.0
ARG MORCA_VERSION=0.31.0
ARG MORCA_RUNTIME=ghcr.io/morca-dev/morca-runtime
FROM ${MORCA_RUNTIME}:${MORCA_VERSION} AS runtime

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ image:
- `MORCA_COMMIT` and `MORCA_SOURCE` in the environment, from build args.

morca's own `morca`, `morca_maintain`, the UI and the rest are the runtime
image's: `ghcr.io/morca-dev/morca-runtime:0.27.0`, the morca-mojo pixi.toml
image's: `ghcr.io/morca-dev/morca-runtime:0.31.0`, the morca-mojo pixi.toml
pins. Moving to a new morca moves three pins together: pixi.toml's
`morca-mojo` revision, Cargo.toml's `morca-worker` tag and
Dockerfile.morca's `MORCA_VERSION`.
Expand Down
14 changes: 11 additions & 3 deletions pipelines/hindcast.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,6 @@ from std.os import getenv
from std.sys import argv

from dista import Seq
from iceberg.schema import Schema
from iceberg.transforms import PartitionField, PartitionSpec, parse_transform
from morca import (
Cron,
Ctx,
Expand All @@ -53,7 +51,17 @@ from morca import (
)
from morca.civil import days_from_civil, now
from morca.rust_step import rust_step
from morca.tables import append, ensure_table, load_dataset, scan, table
from morca.tables import (
PartitionField,
PartitionSpec,
Schema,
append,
ensure_table,
load_dataset,
parse_transform,
scan,
table,
)

from skill import (
CITIES_SCHEMA,
Expand Down
3,164 changes: 1,390 additions & 1,774 deletions pixi.lock

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ version = "0.1.0"
preview = ["pixi-build"]

# morca as `pixi shelf add morca-mojo` pins it: by the revision of its
# release (v0.27.0), which brings dista, iceberg and the rest at the
# release (v0.31.0), which brings dista, iceberg and the rest at the
# revisions it was built against. Cargo.toml's morca-worker is the same tag;
# move the two together.
[dependencies]
mojo-compiler = "==1.2.0"
morca-mojo = { git = "https://github.com/morca-dev/morca.git", rev = "e3587614f6ccc530887481a7c8b3eaf63d7a1af7" }
morca-mojo = { git = "https://github.com/morca-dev/morca.git", rev = "e36173c96db93f839450a852f7262c536da4bd4e" }

[tasks]
# The schema as a Mojo module (scripts/embed-json.sh), then the pipeline.
Expand Down
71 changes: 36 additions & 35 deletions src/skill.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,17 @@ Values are in kelvin, so an error is one in °C too.

from std.math import cos, floor, pi, sqrt

from iceberg.batch import ColumnBuilder, batch_of
from iceberg.scan import TableScan
from iceberg.schema import Schema
from iceberg.transforms import PartitionField, PartitionSpec, parse_transform
from iceberg.values import Datum
from parquet import RecordBatch

from morca.civil import Civil
from morca.tables import (
BatchBuilder,
PartitionField,
PartitionSpec,
Schema,
TableScan,
parse_transform,
)

comptime NI = 360
"""Longitudes, 0°E to 359°E."""
Expand Down Expand Up @@ -213,21 +216,19 @@ def score(forecast: Field, analysis: Field) raises -> Skill:
struct Scores(Movable):
"""The rows the scoring step writes, gathered before one commit each."""

var skill_schema: Schema
var cities_schema: Schema
var skill: List[ColumnBuilder]
var city: List[ColumnBuilder]
var skill: BatchBuilder
var city: BatchBuilder
var sk: List[Int]
"""`skill`'s columns, in the schema's order."""
var ci: List[Int]
"""`city`'s columns, in the schema's order."""
var rows: Int

def __init__(out self) raises:
self.skill_schema = Schema.parse(SKILL_SCHEMA)
self.cities_schema = Schema.parse(CITIES_SCHEMA)
self.skill = List[ColumnBuilder]()
self.city = List[ColumnBuilder]()
for i in range(1, 8):
self.skill.append(ColumnBuilder.of(self.skill_schema, i))
for i in range(1, 10):
self.city.append(ColumnBuilder.of(self.cities_schema, i))
self.skill = BatchBuilder(Schema.parse(SKILL_SCHEMA))
self.city = BatchBuilder(Schema.parse(CITIES_SCHEMA))
self.sk = self.skill.roots.copy()
self.ci = self.city.roots.copy()
self.rows = 0

def add(
Expand All @@ -239,31 +240,31 @@ struct Scores(Movable):
analysis: Field,
) raises -> Skill:
var s = score(forecast, analysis)
self.skill[0].add_int(valid_day)
self.skill[1].add_int(cycle_day)
self.skill[2].add_int(Int64(lead))
self.skill[3].add_int(Int64(s.points))
self.skill[4].add_float(s.bias)
self.skill[5].add_float(s.mae)
self.skill[6].add_float(s.rmse)
self.skill.add_int(self.sk[0], valid_day)
self.skill.add_int(self.sk[1], cycle_day)
self.skill.add_int(self.sk[2], Int64(lead))
self.skill.add_int(self.sk[3], Int64(s.points))
self.skill.add_float(self.sk[4], s.bias)
self.skill.add_float(self.sk[5], s.mae)
self.skill.add_float(self.sk[6], s.rmse)
self.rows += 1
for c in cities():
var p = point_of(c.lat, c.lon)
if not forecast.has(p) or not analysis.has(p):
continue
self.city[0].add_int(valid_day)
self.city[1].add_int(cycle_day)
self.city[2].add_int(Int64(lead))
self.city[3].add(Datum.string_(c.name))
self.city[4].add_float(lat_of(p))
self.city[5].add_float(lon_of(p))
self.city[6].add_float(forecast.t[p])
self.city[7].add_float(analysis.t[p])
self.city[8].add_float(forecast.t[p] - analysis.t[p])
self.city.add_int(self.ci[0], valid_day)
self.city.add_int(self.ci[1], cycle_day)
self.city.add_int(self.ci[2], Int64(lead))
self.city.add_str(self.ci[3], c.name)
self.city.add_float(self.ci[4], lat_of(p))
self.city.add_float(self.ci[5], lon_of(p))
self.city.add_float(self.ci[6], forecast.t[p])
self.city.add_float(self.ci[7], analysis.t[p])
self.city.add_float(self.ci[8], forecast.t[p] - analysis.t[p])
return s^

def skill_batches(mut self) raises -> List[RecordBatch]:
return [batch_of(self.skill)]
return [self.skill.batch()]

def city_batches(mut self) raises -> List[RecordBatch]:
return [batch_of(self.city)]
return [self.city.batch()]
Loading