Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: magmalake/customs@v0.3.0
- uses: magmalake/customs@v0.3.1

# One check for every Linux job above: each passed, or, for a prose-only
# PR, was skipped. The jobs' own checks still report too, so the ruleset's
Expand Down
53 changes: 30 additions & 23 deletions pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,29 +10,35 @@ preview = ["pixi-build"]
platforms = ["osx-arm64", "linux-64", "linux-aarch64"]

[tasks]
test-codec = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_codec.mojo -o build/test-codec && ./build/test-codec" }
test-frame = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_frame.mojo -o build/test-frame && ./build/test-frame" }
test-auth = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_auth.mojo -o build/test-auth && ./build/test-auth" }
test-runtime = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_runtime.mojo -o build/test-runtime && ./build/test-runtime" }
test-batches = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_batches.mojo -o build/test-batches && ./build/test-batches" }
test-free = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_free.mojo -o build/test-free && ./build/test-free" }
test-actors = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_actors.mojo -o build/test-actors && ./build/test-actors" }
test-arity = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_arity.mojo -o build/test-arity && ./build/test-arity" }
test-shuffle = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_shuffle.mojo -o build/test-shuffle && ./build/test-shuffle" }
test-take = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_take.mojo -o build/test-take && ./build/test-take" }
test-admission = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_admission.mojo -o build/test-admission && ./build/test-admission" }
test-head-queue = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_head_queue.mojo -o build/test-head-queue && ./build/test-head-queue" }
# The unit tests, tests/test_*.mojo, built once into one binary and each run
# as its own process (scripts/test.sh); `test-codec` and the others below run
# one file from it. test-libc-abi and test-head-loop stay builds of their own
# (scripts/test.sh says why).
test-units = { cmd = "sh scripts/test.sh" }
test-codec = { cmd = "sh scripts/test.sh test_codec" }
test-frame = { cmd = "sh scripts/test.sh test_frame" }
test-auth = { cmd = "sh scripts/test.sh test_auth" }
test-runtime = { cmd = "sh scripts/test.sh test_runtime" }
test-batches = { cmd = "sh scripts/test.sh test_batches" }
test-free = { cmd = "sh scripts/test.sh test_free" }
test-actors = { cmd = "sh scripts/test.sh test_actors" }
test-arity = { cmd = "sh scripts/test.sh test_arity" }
test-shuffle = { cmd = "sh scripts/test.sh test_shuffle" }
test-take = { cmd = "sh scripts/test.sh test_take" }
test-admission = { cmd = "sh scripts/test.sh test_admission" }
test-head-queue = { cmd = "sh scripts/test.sh test_head_queue" }
# Built on its own: its `signal` declaration clashes with dista's (scripts/test.sh).
test-head-loop = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_head_loop.mojo -o build/test-head-loop && ./build/test-head-loop" }
test-fetch = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_fetch.mojo -o build/test-fetch && ./build/test-fetch" }
test-spill = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_spill.mojo -o build/test-spill && ./build/test-spill" }
test-timeline = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_timeline.mojo -o build/test-timeline && ./build/test-timeline" }
test-cancel = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_cancel.mojo -o build/test-cancel && ./build/test-cancel" }
test-head-drivers = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_head_drivers.mojo -o build/test-head-drivers && ./build/test-head-drivers" }
test-scale = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_scale.mojo -o build/test-scale && ./build/test-scale" }
test-executors = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_executors.mojo -o build/test-executors && ./build/test-executors" }
test-slots = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_slots.mojo -o build/test-slots && ./build/test-slots" }
test-fetch = { cmd = "sh scripts/test.sh test_fetch" }
test-spill = { cmd = "sh scripts/test.sh test_spill" }
test-timeline = { cmd = "sh scripts/test.sh test_timeline" }
test-cancel = { cmd = "sh scripts/test.sh test_cancel" }
test-head-drivers = { cmd = "sh scripts/test.sh test_head_drivers" }
test-scale = { cmd = "sh scripts/test.sh test_scale" }
test-executors = { cmd = "sh scripts/test.sh test_executors" }
test-slots = { cmd = "sh scripts/test.sh test_slots" }
test-libc-abi = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_libc_abi.mojo -o build/test-libc-abi && ./build/test-libc-abi" }
test-sizing = { cmd = "scripts/mojo-build.sh -I src $DEP_INC tests/test_sizing.mojo -o build/test-sizing && ./build/test-sizing" }
test-sizing = { cmd = "sh scripts/test.sh test_sizing" }
test-cluster = { cmd = "tests/cluster.sh" }
test-restart = { cmd = "tests/head_restart.sh" }
test-locality = { cmd = "tests/locality.sh" }
Expand All @@ -46,9 +52,10 @@ test-cli = { cmd = "tests/cli.sh" }
test-chaos = { cmd = "CHAOS_FAST=1 tests/chaos.sh" }
test-kind = { cmd = "tests/kind.sh" }
# The test binaries built several at once before the tasks that copy them
# (scripts/build-tests.sh, scripts/mojo-build.sh).
# (scripts/build-tests.sh, scripts/mojo-build.sh): the unit tests' one
# binary, test-libc-abi's, test-head-loop's and build/dista.
build-tests = { cmd = "scripts/build-tests.sh" }
test = { depends-on = ["build-tests", "test-codec", "test-frame", "test-auth", "test-runtime", "test-batches", "test-free", "test-take", "test-timeline", "test-scale", "test-executors", "test-slots", "test-sizing", "test-libc-abi", "test-head-drivers", "test-actors", "test-arity", "test-shuffle", "test-spill", "test-fetch", "test-admission", "test-head-loop", "test-head-queue", "test-cancel", "test-cluster", "test-restart", "test-locality", "test-compile-fail", "test-metrics", "test-drain", "test-peer-loss", "test-peer-connect", "test-autoscale", "test-cli", "test-chaos"] }
test = { depends-on = ["build-tests", "test-units", "test-libc-abi", "test-head-loop", "test-cluster", "test-restart", "test-locality", "test-compile-fail", "test-metrics", "test-drain", "test-peer-loss", "test-peer-connect", "test-autoscale", "test-cli", "test-chaos"] }
cli = { cmd = "scripts/mojo-build.sh -I src $DEP_INC cli/main.mojo -o build/dista" }
bench = { cmd = "scripts/mojo-build.sh -I src $DEP_INC bench/fanout.mojo -o build/bench-fanout && ./build/bench-fanout" }
bench-slots = { cmd = "scripts/mojo-build.sh -I src $DEP_INC bench/slots.mojo -o build/bench-slots && ./build/bench-slots" }
Expand Down
16 changes: 7 additions & 9 deletions scripts/build-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,15 @@
# One `mojo build` keeps under two cores busy; this runs one per two cores.
#
# Usage: scripts/build-tests.sh [source.mojo...]
# With no sources: every tests/test_*.mojo the default environment builds
# (not the data environment's Parquet and Iceberg tests), and cli/main.mojo,
# With no sources: the unit tests' one binary (scripts/test.sh --build),
# tests/test_libc_abi.mojo and tests/test_head_loop.mojo, which are built
# on their own, and cli/main.mojo,
# which three of the shell tests build.
set -eu
if [ "$#" -eq 0 ]; then
for f in tests/test_*.mojo; do
case "$f" in
tests/test_parquet_tasks.mojo | tests/test_iceberg_scan.mojo) ;;
*) set -- "$@" "$f" ;;
esac
done
set -- "$@" cli/main.mojo
set -- tests/test_libc_abi.mojo tests/test_head_loop.mojo cli/main.mojo
sh scripts/test.sh --build &
units=$!
fi
cores=$(getconf _NPROCESSORS_ONLN 2> /dev/null || echo 2)
jobs=$((cores / 2 + 1))
Expand All @@ -26,3 +23,4 @@ printf '%s\n' "$@" | xargs -n 1 -P "$jobs" sh -c '
cli/main.mojo) exec scripts/mojo-build.sh -I src ${DEP_INC:--I src} "$1" -o build/dista ;;
*) exec scripts/mojo-build.sh -I src $DEP_INC "$1" -o "build/$(basename "$1" .mojo | tr _ -)" ;;
esac' sh
[ -z "${units:-}" ] || wait "$units"
144 changes: 144 additions & 0 deletions scripts/test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
#!/bin/sh
# The unit tests, built once and run from one binary:
#
# scripts/test.sh [-k] [TEST...] # pixi run test-units [test_codec ...]
# scripts/test.sh --build # only build it (scripts/build-tests.sh)
#
# Every tests/test_*.mojo but the ones below goes into
# build/tests/multicall-tests, and each test file then runs as its own
# process, build/tests/<test>, a link to it. The library code the tests share
# is compiled once, not once per test file (census's scripts/test.sh, which
# this follows). With TEST arguments (file names, with or without tests/ and
# .mojo) only those run; the binary still has them all. -k keeps going after
# a failure.
#
# Not in it, each built on its own:
# - tests/test_head_loop.mojo (`pixi run test-head-loop`): its main ignores
# SIGPIPE with `signal(13, SIG_IGN)`, an integer for the handler, where
# dista's own code declares `signal` with a function (dista.signals). Mojo
# allows one signature per C symbol in a program, so the two can't be in
# one binary.
# - tests/test_libc_abi.mojo (`pixi run test-libc-abi`): it declares libc's
# socket calls as Mosaad-M/tcp does, in one program with dista's own, and
# must fail to build if the two disagree. In a binary with every test, a
# clash would break all of them, or be hidden by another file's declaration.
# - tests/test_parquet_tasks.mojo and tests/test_iceberg_scan.mojo: they need
# the data environment (`pixi run -e data test-parquet`, `test-iceberg`).
#
# The tests are registered by name here, each `def test_...` at column 0,
# not found with `__functions_in_module()`: in a module that is imported,
# rather than compiled as the main file, that list comes back incomplete
# (Mojo 1.2.0), and a test file would pass having run nothing. The count the
# binary reports is checked against the functions found here. A test file's
# `main` is therefore not called: it may only run the tests, and what a file
# needs done around them goes in `def before_tests() raises` and
# `def after_tests() raises`, called here and by its main (as in census's
# tests/test_fetch.mojo).
#
# One file alone, without this: scripts/mojo-build.sh -I src $DEP_INC
# tests/test_codec.mojo -o build/test-codec && ./build/test-codec
# Run inside `pixi run`.
set -eu
keep=""
build_only=""
[ "${1:-}" = -k ] && { keep=1; shift; }
[ "${1:-}" = --build ] && { build_only=1; shift; }
dir=build/tests
gen=build/multicall-gen/tests
mkdir -p "$gen" "$dir"
main=$gen/multicall_tests_main.mojo
tests=""
for f in tests/test_*.mojo; do
case "$f" in
tests/test_head_loop.mojo | tests/test_libc_abi.mojo | tests/test_parquet_tasks.mojo | tests/test_iceberg_scan.mojo) ;;
*) tests="$tests $f" ;;
esac
done
functions() { sed -n 's/^def \(test_[A-Za-z0-9_]*\)(.*/\1/p' "$1"; }
{
echo '"""Generated by scripts/test.sh: the tests of the file basename(argv[0])'
echo 'names."""'
echo
echo 'from std.sys import argv'
echo 'from std.testing import TestSuite'
echo
for f in $tests; do
m=$(basename "$f" .mojo)
for t in $(functions "$f"); do echo "from $m import $t as ${m}__$t"; done
if grep -q '^def before_tests()' "$f"; then
echo "from $m import before_tests as ${m}__before_tests"
echo "from $m import after_tests as ${m}__after_tests"
fi
done
echo
echo
echo 'def main() raises:'
echo ' var me = String(argv()[0])'
echo ' var slash = me.rfind("/")'
echo ' var name = String(me[byte = slash + 1 :]) if slash >= 0 else me'
kw=if
for f in $tests; do
m=$(basename "$f" .mojo)
list=""
for t in $(functions "$f"); do list="$list${m}__$t, "; done
[ -n "$list" ] || { echo "test.sh: $f has no test" >&2; exit 1; }
echo " $kw name == \"$m\":"
if grep -q '^def before_tests()' "$f"; then
echo " ${m}__before_tests()"
echo ' var failed = False'
echo ' try:'
echo " TestSuite.discover_tests[($list)]().run()"
echo ' except e:'
echo ' failed = True'
echo ' print(e)'
echo " ${m}__after_tests()"
echo ' if failed:'
echo " raise Error(\"$m failed\")"
else
# Its main is not called, so it may only run the tests.
[ "$(sed -n '/^def main()/,$p' "$f" | grep -vc '^[[:space:]]*$')" = 2 ] || {
echo "test.sh: $f: main does more than run the tests; move that to before_tests() and after_tests()" >&2
exit 1
}
echo " TestSuite.discover_tests[($list)]().run()"
fi
kw=elif
done
echo ' else:'
echo ' raise Error(name, ": not a test of this binary")'
} > "$main.tmp"
# Rewritten only when it changed, so scripts/mojo-build.sh's key holds.
if cmp -s "$main.tmp" "$main"; then rm -f "$main.tmp"; else mv -f "$main.tmp" "$main"; fi
t=$(date +%s)
# shellcheck disable=SC2086
scripts/mojo-build.sh -I src -I tests $DEP_INC "$main" -o "$dir/multicall-tests"
echo "test.sh: built in $(($(date +%s) - t)) s"
for f in $tests; do ln -sf multicall-tests "$dir/$(basename "$f" .mojo)"; done
[ -z "$build_only" ] || exit 0

if [ $# -gt 0 ]; then
tests=""
for a in "$@"; do
a=tests/$(basename "$a" .mojo).mojo
[ -f "$a" ] || { echo "test.sh: no $a" >&2; exit 1; }
tests="$tests $a"
done
fi
failed=""
log=$gen/run.log
for f in $tests; do
m=$(basename "$f" .mojo)
want=$(functions "$f" | wc -l | tr -d ' ')
echo "== $f"
if "$dir/$m" > "$log" 2>&1; then rc=0; else rc=$?; fi
cat "$log"
ran=$(sed -n 's/.* \([0-9][0-9]*\) tests run:.*/\1/p' "$log" | tail -1)
if [ $rc != 0 ]; then
failed="$failed $m"
elif [ "${ran:-0}" != "$want" ]; then
echo "test.sh: $f has $want tests and ran ${ran:-0}" >&2
failed="$failed $m"
fi
[ -z "$failed" ] || [ -n "$keep" ] || break
done
[ -z "$failed" ] || { echo "test.sh: failed:$failed" >&2; exit 1; }
Loading