diff --git a/changes/539-ui-workers.minor.md b/changes/539-ui-workers.minor.md new file mode 100644 index 00000000..1c202eb4 --- /dev/null +++ b/changes/539-ui-workers.minor.md @@ -0,0 +1 @@ +- **UI workers and request log**: `morca ui` serves on `MORCA_UI_WORKERS` threads (the Hetzner stack runs 4) and logs each request. (#539) diff --git a/deploy/hetzner/VERSION b/deploy/hetzner/VERSION index 85e60ed1..7b52f5e5 100644 --- a/deploy/hetzner/VERSION +++ b/deploy/hetzner/VERSION @@ -1 +1 @@ -0.34.0 +0.35.0 diff --git a/deploy/hetzner/server/morca-stack b/deploy/hetzner/server/morca-stack index 0eac3dad..e579631f 100755 --- a/deploy/hetzner/server/morca-stack +++ b/deploy/hetzner/server/morca-stack @@ -484,6 +484,7 @@ MORCA_UI_ADDR=0.0.0.0:8270 MORCA_UI_LOGIN=required MORCA_UI_SECURE_COOKIES=1 MORCA_UI_TRUST_PROXY=1 +MORCA_UI_WORKERS=4 EOS # What machines of a worker pool run: the server's own image. echo "MORCA_WORKER_IMAGE=$image:$tag" diff --git a/docs/ui.md b/docs/ui.md index 10c58004..696e9bcd 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -713,6 +713,22 @@ used. whether it is past it, the snooze form, and the trips with what each tripped on. See [watchdog.md](watchdog.md). +## Workers and the request log + +`morca ui` answers on `MORCA_UI_WORKERS` threads (default 1; the Hetzner +stack's UI runs 4). They share one listening socket, so a new connection goes +to a free thread instead of waiting behind a slow page. Each request except +`/healthz` and the stylesheet is logged as one line on standard output: + +``` +morca ui: GET /runs?status=failed 200 42 ms 172.18.0.5 cf=203.0.113.7 "Mozilla/5.0 ..." +``` + +That's the method, URL, status, milliseconds, and the client address (from +`X-Forwarded-For` with `MORCA_UI_TRUST_PROXY=1`), followed by Cloudflare's +`CF-Connecting-IP` when the request came through a tunnel, and the +User-Agent. + ## Stopping SIGTERM (what `docker stop` sends) or SIGINT (Ctrl-C) drains `morca ui` @@ -724,5 +740,7 @@ exits 0 once none has been for 200 ms. `MORCA_UI_DRAIN_SECONDS` (default 8, under `docker stop`'s 10 s grace) bounds the wait; when it's up the server stops and closes its kept-alive connections, and a request still running half a second later is given up, with exit code 1. A second signal stops -it at once. The worker API on `MORCA_WORKER_ADDR`, if any, ends with the +it at once. On more than one worker, the drain doesn't close the workers' +listening socket, so a new connection keeps being answered until the loop +stops. The worker API on `MORCA_WORKER_ADDR`, if any, ends with the process. `src/morca/_internal/web/drain.mojo` has the details. diff --git a/src/morca/_internal/web/server.mojo b/src/morca/_internal/web/server.mojo index bace8751..fa1347b5 100644 --- a/src/morca/_internal/web/server.mojo +++ b/src/morca/_internal/web/server.mojo @@ -32,9 +32,27 @@ otherwise. `MORCA_WORKER_ADDR` serves the worker API on a listener of its own as well (`morca._internal.api.worker`). SIGTERM or SIGINT drains it: no new connections, `/healthz` 503, the requests in flight answered, then exit 0 within `MORCA_UI_DRAIN_SECONDS` (8; `morca._internal.web.drain`). + +It answers on `MORCA_UI_WORKERS` threads (default 1; the Hetzner stack runs 4), +sharing one listening socket (flare's `FLARE_REUSEPORT_WORKERS=0`, unless +that's set), so a new connection goes to a thread that's free rather than +one a slow page holds up. +Each request but `/healthz` and the stylesheet is logged as one line: method, +path, status, milliseconds, the client (`client`), Cloudflare's +`CF-Connecting-IP` when there is one, and the User-Agent. """ -from flare.http import Handler, HttpServer, Request, Response, ServerConfig +from std.os import getenv, setenv +from std.time import perf_counter_ns + +from flare.http import ( + Handler, + HttpServer, + Request, + Response, + ResponseImpl, + ServerConfig, +) from flare.net import SocketAddr import morca._internal.db.health as health_db from threads import AtomicFlag @@ -304,6 +322,82 @@ struct Ui(Copyable, Handler): return _error(e, json) +def _clip(s: String, n: Int) -> String: + if s.byte_length() <= n: + return s + return String(s[byte=:n], "...") + + +def _log(req: Request, status: Int, start_ns: Int): + var ms = (Int(perf_counter_ns()) - start_ns) // 1_000_000 + var cf = req.headers.get("CF-Connecting-IP") + print( + "morca ui: ", + req.method, + " ", + _clip(req.url, 200), + " ", + status, + " ", + ms, + " ms ", + client(req), + String(" cf=", _clip(cf, 64)) if cf else String(), + ' "', + _clip(req.headers.get("User-Agent"), 120), + '"', + sep="", + flush=True, + ) + + +struct Logged[H: Handler & Copyable](Copyable, Handler): + """`H`, logging each request but `/healthz` and the stylesheet as it's + answered.""" + + comptime BodyType = Self.H.BodyType + + var inner: Self.H + + def __init__(out self, var inner: Self.H): + self.inner = inner^ + + def serve(self, req: Request) raises -> ResponseImpl[Self.BodyType]: + var start = Int(perf_counter_ns()) + var path = _path_of(req.url) + var quiet = path == HEALTHZ or path == STYLESHEET + try: + var resp = self.inner.serve(req) + if not quiet: + _log(req, resp.status, start) + return resp^ + except e: + if not quiet: + _log(req, 500, start) + raise e^ + + +def ui_workers() -> Int: + """`MORCA_UI_WORKERS`, or 1 if it's unset or not a number of at least 1. + One worker's drain refuses new connections at once; several keep + answering them until the loop stops (docs/ui.md#stopping).""" + var raw = getenv("MORCA_UI_WORKERS") + if not raw: + return 1 + try: + var n = Int(raw) + if n >= 1: + return n + except: + pass + print( + "morca ui: MORCA_UI_WORKERS isn't a number of workers:", + raw, + "- using 1", + ) + return 1 + + def read_page(mut db: State, url: String) raises -> Page: """What `GET url` answers, for `morca` commands that print a page's JSON over the database (`morca._internal.remote`).""" @@ -334,7 +428,12 @@ def serve_ui(url: String, addr: String, worker_addr: String = "") raises: worker_api.listen(url, worker_addr) print("morca ui: the worker API on http://", worker_addr, sep="") var drain = Drain(server) - print("morca ui on http://", addr, sep="") - server.serve(drain.counting(Ui(url, drain.draining()))) + var workers = ui_workers() + if workers > 1 and not getenv("FLARE_REUSEPORT_WORKERS"): + # Per-worker listeners would hash some new connections to a worker + # that's busy with a slow page. + _ = setenv("FLARE_REUSEPORT_WORKERS", "0") + print("morca ui on http://", addr, " (", workers, " workers)", sep="") + server.serve(Logged(drain.counting(Ui(url, drain.draining()))), workers) _ = drain^ print("morca ui: stopped") diff --git a/tests/ui_drain.sh b/tests/ui_drain.sh index 57c52f3e..6612a1f5 100755 --- a/tests/ui_drain.sh +++ b/tests/ui_drain.sh @@ -9,7 +9,12 @@ # MORCA_UI_DRAIN_SECONDS; # - SIGTERM with nothing in flight: it exits 0 at once; # - SIGINT with a request that outlasts MORCA_UI_DRAIN_SECONDS: it exits -# (1, the request cut) within the bound plus half a second. +# (1, the request cut) within the bound plus half a second; +# - on 2 workers (MORCA_UI_WORKERS), a new request is answered while a slow +# one waits, both are logged, and SIGTERM still lets the slow one finish. +# +# The first three run on one worker: with several, each worker's connections +# keep being accepted until the drain stops the loop. # # Needs MORCA_DATABASE_URL (pixi run test-ui-drain starts a throwaway server), # curl, and bash (its /dev/tcp holds the kept-alive connection). @@ -36,11 +41,12 @@ fail() { q() { psql "$MORCA_DATABASE_URL" -Atc "$1"; } ./build/morca register ./build/toy >> "$log" 2>&1 || fail "register" -# start DRAIN_SECONDS: morca ui on a free port, once it answers. +# start DRAIN_SECONDS [WORKERS]: morca ui on a free port (on 1 worker +# unless told), once it answers. start() { free_port port base="http://127.0.0.1:$port" - MORCA_UI_DRAIN_SECONDS=$1 MORCA_UI_ADDR="127.0.0.1:$port" ./build/morca ui >> "$log" 2>&1 & + MORCA_UI_WORKERS=${2:-1} MORCA_UI_DRAIN_SECONDS=$1 MORCA_UI_ADDR="127.0.0.1:$port" ./build/morca ui >> "$log" 2>&1 & ui=$! i=0 until curl -fs "$base/" > /dev/null 2>&1; do @@ -154,4 +160,19 @@ kill "$locker" 2> /dev/null || true wait "$locker" 2> /dev/null || true locker= wait "$slow_curl" 2> /dev/null || true +# Two workers: a request gets its answer while the other waits on the lock. +start 5 2 +grep -q "morca ui on http://127.0.0.1:$port (2 workers)" "$log" || fail "no 2 workers message" +lock 3 +slow +fast=$(curl -s -o /dev/null -w '%{http_code} %{time_total}' --max-time 2 "$base/static/morca.css" 2> /dev/null || true) +[ "${fast% *}" = 200 ] || fail "a request beside the slow one got: $fast" +kill -TERM "$ui" +stopped 45 +[ "$code" = 0 ] || fail "exited $code after draining on 2 workers" +wait "$slow_curl" || true +[ "$(cat build/ui-drain-slow.txt)" = 200 ] || fail "the slow request on 2 workers got $(cat build/ui-drain-slow.txt)" +wait "$locker" || true +locker= +grep -Eq '^morca ui: GET /runs 200 [0-9]+ ms 127\.0\.0\.1 "curl/' "$log" || fail "/runs wasn't logged" echo "ui drain: ok"