diff --git a/changes/545-ui-limits.minor.md b/changes/545-ui-limits.minor.md new file mode 100644 index 00000000..38b6f3d7 --- /dev/null +++ b/changes/545-ui-limits.minor.md @@ -0,0 +1 @@ +- **The UI limits each client:** `MORCA_UI_RATE` requests a minute (429), and data reads one at a time per client and `MORCA_UI_DATA_READS` at once (503); the Hetzner stack runs 8 workers at 120 a minute, and log lines start with the time. (#545) diff --git a/deploy/hetzner/VERSION b/deploy/hetzner/VERSION index 7b52f5e5..93d4c1ef 100644 --- a/deploy/hetzner/VERSION +++ b/deploy/hetzner/VERSION @@ -1 +1 @@ -0.35.0 +0.36.0 diff --git a/deploy/hetzner/server/morca-stack b/deploy/hetzner/server/morca-stack index e579631f..6834bc54 100755 --- a/deploy/hetzner/server/morca-stack +++ b/deploy/hetzner/server/morca-stack @@ -484,7 +484,8 @@ MORCA_UI_ADDR=0.0.0.0:8270 MORCA_UI_LOGIN=required MORCA_UI_SECURE_COOKIES=1 MORCA_UI_TRUST_PROXY=1 -MORCA_UI_WORKERS=4 +MORCA_UI_WORKERS=8 +MORCA_UI_RATE=120 EOS # What machines of a worker pool run: the server's own image. echo "MORCA_WORKER_IMAGE=$image:$tag" diff --git a/docs/ui.md b/docs/ui.md index 84413c4f..0bbe62b7 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -716,18 +716,35 @@ tripped on. See [watchdog.md](watchdog.md). ## Workers and the request log `morca ui` answers on `MORCA_UI_WORKERS` threads (default 1; the Hetzner -stack's UI runs 4). They share one listening socket, so a new connection goes +stack's UI runs 8). They share one listening socket, so a new connection goes to a free thread instead of waiting behind a slow page. Each request except `/healthz` and the stylesheet is logged as one line on standard output: ``` -morca ui: GET /runs?status=failed 200 42 ms 172.18.0.5 cf=203.0.113.7 "Mozilla/5.0 ..." +morca ui: 2026-11-18T14:02:03Z GET /runs?status=failed 200 42 ms 172.18.0.5 cf=203.0.113.7 "Mozilla/5.0 ..." ``` -That's the method, URL, status, milliseconds, and the client address (from -`X-Forwarded-For` with `MORCA_UI_TRUST_PROXY=1`), followed by Cloudflare's -`CF-Connecting-IP` when the request came through a tunnel, and the -User-Agent. +That's the time (UTC), method, URL, status, milliseconds, and the client +address (from `X-Forwarded-For` with `MORCA_UI_TRUST_PROXY=1`), followed by +Cloudflare's `CF-Connecting-IP` when the request came through a tunnel, and +the User-Agent. + +## Limits per client + +A data read (a Data tab, `data.csv`, `data.json`) runs `morca_maintain data` +and takes a second or more, and a thread waits for it. So that the other +pages don't queue behind them, the UI caps them: + +| Variable | Default | | +|---|---|---| +| `MORCA_UI_RATE` | 0 (off) | requests one client may send a minute; the Hetzner stack sets 120 | +| `MORCA_UI_DATA_READS` | half the workers, at least 1 | data reads running at once, for all clients; one client may have one | + +A client over its rate gets 429 with `Retry-After` (the seconds left of its +minute). A data read past the cap, or a second one from a client with one +running, gets 503 with `Retry-After: 5`. Requests with a session cookie or an +`Authorization` header (people logged in, and scripts) aren't counted, nor +are `/healthz`, the stylesheet, `/robots.txt` and the worker API. `/robots.txt` disallows every crawler, and each answer carries `X-Robots-Tag: noindex, nofollow`: run, snapshot and data pages are endless diff --git a/pixi.toml b/pixi.toml index 794dda49..27155d7f 100644 --- a/pixi.toml +++ b/pixi.toml @@ -215,6 +215,7 @@ test-weekly = { cmd = "scripts/with-postgres.sh tests/weekly.sh" } # morca.toml (morca.toml, morca.config): the parser, what register reads from # the file and records, config show and export, and the env on each runner. test-toml = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-toml" } +test-limit = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-limit" } test-upgrade = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-upgrade" } test-config = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-config && scripts/with-postgres.sh tests/config.sh" } # Fills (morca.fill): a pipeline filling its range by itself, within limits. @@ -299,7 +300,7 @@ test-executor-core = { depends-on = ["build-shared", "test-sample", "test-steps" test-executor-workers = { depends-on = ["build-shared", "test-ec2", "test-autoscale", "test-no-workers", "test-pool", "test-hcloud", "test-pool-scale", "test-pool-scale-ec2", "test-pool-image", "test-run-cluster", "test-secrets", "test-placement", "test-grants", "test-pressure", "test-lost-workers"] } test-executor = { depends-on = ["test-executor-core", "test-executor-workers"] } # The scheduler, the UI, and what a registration and its code control: -test-scheduler-ui = { depends-on = ["build-shared", "test-toml", "test-upgrade", "test-config", "test-fill", "test-fill-version", "test-fill-requests", "test-priority", "test-pool-fills", "test-run-locks", "test-metrics", "test-terms", "test-scheduler", "test-ui", "test-fleet", "test-system", "test-actions", "test-calendar", "test-ui-login", "test-ui-drain", "test-remote", "test-worker-api", "test-pause", "test-older-binary", "test-watchdog", "test-notify", "test-versions", "test-manual-run", "test-new-code", "test-defer", "test-dead-runs", "test-adopt", "test-overlap", "test-outdated", "test-concurrency", "test-one-run-per-key", "test-hourly", "test-weekly", "test-sense-errors", "test-stop", "test-binary-timeouts"] } +test-scheduler-ui = { depends-on = ["build-shared", "test-toml", "test-upgrade", "test-config", "test-fill", "test-fill-version", "test-fill-requests", "test-priority", "test-pool-fills", "test-run-locks", "test-metrics", "test-terms", "test-scheduler", "test-ui", "test-fleet", "test-system", "test-actions", "test-calendar", "test-ui-login", "test-ui-drain", "test-remote", "test-worker-api", "test-pause", "test-older-binary", "test-watchdog", "test-limit", "test-notify", "test-versions", "test-manual-run", "test-new-code", "test-defer", "test-dead-runs", "test-adopt", "test-overlap", "test-outdated", "test-concurrency", "test-one-run-per-key", "test-hourly", "test-weekly", "test-sense-errors", "test-stop", "test-binary-timeouts"] } test = { depends-on = ["test-executor", "test-scheduler-ui"] } toy = { cmd = "scripts/mojo-build.sh -I src $DEP_INC examples/toy.mojo -o build/toy" } fanout = { cmd = "scripts/mojo-build.sh -I src $DEP_INC examples/fanout.mojo -o build/fanout" } diff --git a/scripts/unit-tests.sh b/scripts/unit-tests.sh index cd4fd79f..46d7b273 100755 --- a/scripts/unit-tests.sh +++ b/scripts/unit-tests.sh @@ -85,6 +85,7 @@ scheduler-ui:tests/test_terms.mojo:test-terms scheduler-ui:tests/test_remote.mojo:test-remote scheduler-ui:tests/test_worker.mojo:test-worker-api scheduler-ui:tests/test_watchdog.mojo:test-watchdog +scheduler-ui:tests/test_limit.mojo:test-limit data-tables:tests/test_tables.mojo:test-tables data-tables:tests/test_lake_batch.mojo:test-lake-batch data-tables:tests/test_replace.mojo:test-replace diff --git a/src/morca/_internal/web/limit.mojo b/src/morca/_internal/web/limit.mojo new file mode 100644 index 00000000..0869bcde --- /dev/null +++ b/src/morca/_internal/web/limit.mojo @@ -0,0 +1,290 @@ +"""What one client may ask of `morca ui` (#545): a number of requests a +minute, and how many of the expensive data reads (a Data tab, `data.csv`, +`data.json`) it and everyone together may have running at once. + +A data read runs `morca_maintain data` and takes a second or more (more +under load), and a worker thread waits for it, so a crawler that asks for +many at once holds up every page behind them. The cap keeps threads free +for the rest. + +| Variable | Default | | +|---|---|---| +| `MORCA_UI_RATE` | 0 (off) | requests a client may send a minute; the Hetzner stack sets 120 | +| `MORCA_UI_DATA_READS` | half the workers, at least 1 | data reads running at once, for all clients | + +A client is the address `morca._internal.web.auth.client` gives (behind a +proxy, the first of `X-Forwarded-For`). One over its rate is answered 429 +with `Retry-After` (the seconds left of its minute); a data read past the +cap, or a second one from a client that has one running, 503 with +`Retry-After: 5`. Neither limit applies to `/healthz`, the stylesheet, +`/robots.txt` or the worker API, nor to a request with a session cookie or +an `Authorization` header: people logged in and scripts aren't counted. + +The counts live in one table, behind a mutex, that every worker thread +shares; minutes that have ended are dropped from it as it grows. +""" + +from std.ffi import external_call +from std.memory.alloc import unsafe_alloc +from std.os import getenv + +from flare.http import Request, Response +from threads import Mutex + +from morca._internal.auth import SESSION_COOKIE +from morca._internal.web.auth import client, cookie + +comptime WINDOW_SECONDS = 60 +comptime BUSY_RETRY_SECONDS = 5 +comptime _PRUNE_AT = 4096 +"""Clients in the table past which minutes that have ended are dropped +(at least; `ClientTable.prune_at`).""" + + +def _now() -> Int: + return Int(external_call["time", Int64](Int(0))) + + +def _env_count(name: StringSlice, dflt: Int) -> Int: + var raw = getenv(String(name)) + if not raw: + return dflt + try: + var n = Int(raw) + if n >= 0: + return n + except: + pass + print("morca ui:", name, "isn't a count:", raw, "- using", dflt) + return dflt + + +def ui_rate() -> Int: + """`MORCA_UI_RATE`: requests a client may send a minute; 0 is no limit.""" + return _env_count("MORCA_UI_RATE", 0) + + +def ui_data_reads(workers: Int) -> Int: + """`MORCA_UI_DATA_READS`, or half of `workers` (at least 1).""" + return max(1, _env_count("MORCA_UI_DATA_READS", max(1, workers // 2))) + + +def is_data_read(url: String) -> Bool: + """Whether `url` asks for a data read: `/datasets/?tab=data`, + `/datasets//data.csv` or `/datasets//data.json`.""" + if not url.startswith("/datasets/"): + return False + var q = url.find("?") + var path = String(url[byte=:q]) if q >= 0 else url + if path.endswith("/data.csv") or path.endswith("/data.json"): + return True + if q < 0: + return False + for part in String(url[byte = q + 1 :]).split("&"): + if part == "tab=data": + return True + return False + + +def exempt(req: Request, path: String) -> Bool: + """Whether neither limit applies to `req` (the module doc).""" + if ( + path == "/healthz" + or path == "/robots.txt" + or path.startswith("/static/") + or path.startswith("/api/worker/") + ): + return True + if req.headers.get("Authorization"): + return True + return Bool(cookie(req, SESSION_COOKIE)) + + +@fieldwise_init +struct _Client(Copyable, Movable): + var minute: Int + """When its current minute started, in seconds since the epoch.""" + var count: Int + """Requests it has sent in that minute.""" + var reads: Int + """Data reads of its that are running.""" + + +struct ClientTable(Movable): + var lock: Mutex + var clients: Dict[String, _Client] + var reads: Int + """Data reads running, for all clients.""" + var prune_at: Int + """The number of clients at which the next prune runs: twice what the + last one left, so a busy minute doesn't scan the table on every + request.""" + + def __init__(out self) raises: + self.lock = Mutex() + self.clients = Dict[String, _Client]() + self.reads = 0 + self.prune_at = _PRUNE_AT + + +comptime ADMIT = 0 +comptime TOO_MANY = 1 +comptime BUSY = 2 + + +@fieldwise_init +struct Verdict(Copyable, Movable): + """What `admit` decided: `ADMIT`, or why not and when to come back.""" + + var kind: Int + var retry_after: Int + + +def _prune(mut t: ClientTable, now: Int): + var stale = List[String]() + for e in t.clients.items(): + if e.value.reads == 0 and now - e.value.minute >= WINDOW_SECONDS: + stale.append(e.key) + for k in stale: + try: + _ = t.clients.pop(k) + except: + pass + + +def admit( + mut t: ClientTable, + who: String, + is_read: Bool, + rate: Int, + max_reads: Int, + now: Int, +) -> Verdict: + """Counts a request from `who` (a data read if `read`) and says whether + it may go on. The caller holds `t.lock`; one admitted as a read must be + given back with `release`.""" + if len(t.clients) >= t.prune_at: + _prune(t, now) + t.prune_at = max(_PRUNE_AT, 2 * len(t.clients)) + var c = t.clients.get(who, _Client(now, 0, 0)) + if now - c.minute >= WINDOW_SECONDS: + c.minute = now + c.count = 0 + c.count += 1 + if rate > 0 and c.count > rate: + var left = max(1, WINDOW_SECONDS - (now - c.minute)) + t.clients[who] = c^ + return Verdict(TOO_MANY, left) + if is_read: + if c.reads >= 1 or t.reads >= max_reads: + t.clients[who] = c^ + return Verdict(BUSY, BUSY_RETRY_SECONDS) + c.reads += 1 + t.reads += 1 + t.clients[who] = c^ + return Verdict(ADMIT, 0) + + +def release(mut t: ClientTable, who: String): + """Gives back a data read `admit` let `who` start. The caller holds + `t.lock`.""" + t.reads = max(0, t.reads - 1) + try: + var c = t.clients[who].copy() + c.reads = max(0, c.reads - 1) + t.clients[who] = c^ + except: + pass + + +def _refusal(v: Verdict) -> Response: + var status = 429 if v.kind == TOO_MANY else 503 + var body = String( + "too many requests: try again in ", + v.retry_after, + " s\n", + ) if v.kind == TOO_MANY else String( + "the data views are busy: try again in ", + v.retry_after, + " s\n", + ) + var resp = Response( + status=status, + reason="Too Many Requests" if v.kind + == TOO_MANY else "Service Unavailable", + body=List[UInt8](body.as_bytes()), + ) + try: + resp.headers.set("Content-Type", "text/plain; charset=utf-8") + resp.headers.set("Retry-After", String(v.retry_after)) + resp.headers.set("Cache-Control", "no-store") + except: + pass + return resp^ + + +comptime _TablePtr = Pointer[ClientTable, MutUntrackedOrigin] + + +@fieldwise_init +struct Gate(Copyable, ImplicitlyCopyable, Movable): + """What a handler on any worker thread checks the limits with: the + `Limits`' table, which must outlive it, and the limits.""" + + var table: Int + var rate: Int + var max_reads: Int + + def check(self, req: Request, path: String) -> Optional[Response]: + """The refusal for `req`, or None if it may go on. One that goes on + as a data read (`is_data_read`) must be given back with `done`.""" + if exempt(req, path): + return None + ref t = _TablePtr(unsafe_from_address=self.table)[] + t.lock.lock() + var v = admit( + t, + client(req), + is_data_read(req.url), + self.rate, + self.max_reads, + _now(), + ) + t.lock.unlock() + if v.kind != ADMIT: + return _refusal(v) + return None + + def done(self, req: Request, path: String): + """Gives back the data read `check` let `req` start, if it was one.""" + if exempt(req, path) or not is_data_read(req.url): + return + ref t = _TablePtr(unsafe_from_address=self.table)[] + t.lock.lock() + release(t, client(req)) + t.lock.unlock() + + +struct Limits(Movable): + """The table every `Gate` it gives out shares; it must outlive the + server whose handlers use them.""" + + var _table: Int + var rate: Int + var max_reads: Int + + def __init__(out self, workers: Int) raises: + var p = unsafe_alloc[ClientTable](1) + p.unsafe_write(ClientTable()) + self._table = Int(p) + self.rate = ui_rate() + self.max_reads = ui_data_reads(workers) + + def gate(self) -> Gate: + return Gate(self._table, self.rate, self.max_reads) + + def __deinit__(deinit self): + # The server that used the table has stopped. + var p = _TablePtr(unsafe_from_address=self._table) + p.unsafe_deinit_pointee() + p.unsafe_free() diff --git a/src/morca/_internal/web/server.mojo b/src/morca/_internal/web/server.mojo index 37bbcd2b..c115f318 100644 --- a/src/morca/_internal/web/server.mojo +++ b/src/morca/_internal/web/server.mojo @@ -37,8 +37,13 @@ It answers on `MORCA_UI_WORKERS` threads (default 1; the Hetzner stack runs 4), sharing one listening socket (flare's `FLARE_REUSEPORT_WORKERS=0`, unless that's set), so a new connection goes to a thread that's free rather than one a slow page holds up. -Each request but `/healthz` and the stylesheet is logged as one line: method, -path, status, milliseconds, the client (`client`), Cloudflare's +A client may send `MORCA_UI_RATE` requests a minute (off unless set; the +Hetzner stack sets 120), and the data reads, which take a second or more, +run `MORCA_UI_DATA_READS` at once (half the workers), one per client, so +the other pages don't wait behind them; people logged in and scripts aren't +limited (`morca._internal.web.limit`). +Each request but `/healthz` and the stylesheet is logged as one line: the +time (UTC, to the second), method, path, status, milliseconds, the client (`client`), Cloudflare's `CF-Connecting-IP` when there is one, and the User-Agent. """ @@ -61,6 +66,7 @@ from morca._internal.api import worker as worker_api from morca._internal.auth import audit, login_required from morca._internal.html import Html, query_param, url_encode from morca._internal.json import quote +from morca._internal.logs import utc_now from morca._internal.state import State from morca._internal.ui.coverage import COVERAGE_STYLE from morca._internal.ui.data_chart import CHART_STYLE @@ -82,6 +88,7 @@ from morca._internal.web.auth import ( token_refused, ) from morca._internal.web.drain import Drain +from morca._internal.web.limit import Gate, Limits from morca._internal.web.page import Page, respond from morca._internal.web.router import ( ACTION, @@ -243,14 +250,29 @@ struct Ui(Copyable, Handler): var table: List[Route] var draining: AtomicFlag """Set once a drain has started (`morca._internal.web.drain.Drain.draining`).""" + var gate: Gate + """The limits per client (`morca._internal.web.limit`).""" - def __init__(out self, url: String, draining: AtomicFlag): + def __init__(out self, url: String, draining: AtomicFlag, gate: Gate): self.url = url self.table = routes() self.draining = draining + self.gate = gate def serve(self, req: Request) raises -> Response: var path = _path_of(req.url) + var refused = self.gate.check(req, path) + if refused: + return refused.take() + try: + var resp = self._answer(req, path) + self.gate.done(req, path) + return resp^ + except e: + self.gate.done(req, path) + raise e^ + + def _answer(self, req: Request, path: String) raises -> Response: var post = req.method == "POST" if path == STYLESHEET: return _stylesheet() @@ -349,6 +371,8 @@ def _log(req: Request, status: Int, start_ns: Int): var cf = req.headers.get("CF-Connecting-IP") print( "morca ui: ", + utc_now(), + " ", req.method, " ", _clip(req.url, 200), @@ -454,7 +478,25 @@ def serve_ui(url: String, addr: String, worker_addr: String = "") raises: # Per-worker listeners would hash some new connections to a worker # that's busy with a slow page. _ = setenv("FLARE_REUSEPORT_WORKERS", "0") - print("morca ui on http://", addr, " (", workers, " workers)", sep="") - server.serve(Logged(drain.counting(Ui(url, drain.draining()))), workers) + var limits = Limits(workers) + print( + "morca ui on http://", + addr, + " (", + workers, + " workers, ", + limits.max_reads, + " data reads at once", + String( + ", ", limits.rate, " requests a minute a client" + ) if limits.rate else String(), + ")", + sep="", + ) + server.serve( + Logged(drain.counting(Ui(url, drain.draining(), limits.gate()))), + workers, + ) _ = drain^ + _ = limits^ print("morca ui: stopped") diff --git a/tests/test_limit.mojo b/tests/test_limit.mojo new file mode 100644 index 00000000..1231103d --- /dev/null +++ b/tests/test_limit.mojo @@ -0,0 +1,78 @@ +"""The UI's limits per client (`morca._internal.web.limit`): which URLs are +data reads, a client's minute, and the cap on data reads at once.""" + +from std.testing import TestSuite, assert_equal, assert_false, assert_true + +from morca._internal.web.limit import ( + ADMIT, + BUSY, + TOO_MANY, + ClientTable, + admit, + is_data_read, + release, +) + + +def test_data_reads() raises: + assert_true(is_data_read("/datasets/mojo.repos?tab=data")) + assert_true(is_data_read("/datasets/mojo.repos?snapshot=1&tab=data&n=5")) + assert_true(is_data_read("/datasets/mojo.repos/data.csv?snapshot=1")) + assert_true(is_data_read("/datasets/mojo.repos/data.json")) + assert_false(is_data_read("/datasets/mojo.repos")) + assert_false(is_data_read("/datasets/mojo.repos?tab=database")) + assert_false(is_data_read("/datasets/mojo.repos/snapshots/1")) + assert_false(is_data_read("/runs?tab=data")) + + +def test_rate_per_minute() raises: + var t = ClientTable() + for _ in range(3): + assert_equal(admit(t, "a", False, 3, 2, 1000).kind, ADMIT) + var v = admit(t, "a", False, 3, 2, 1010) + assert_equal(v.kind, TOO_MANY) + assert_equal(v.retry_after, 50) + # Another client has its own minute. + assert_equal(admit(t, "b", False, 3, 2, 1010).kind, ADMIT) + # A new minute starts the count again. + assert_equal(admit(t, "a", False, 3, 2, 1060).kind, ADMIT) + + +def test_no_rate() raises: + var t = ClientTable() + for _ in range(1000): + assert_equal(admit(t, "a", False, 0, 2, 1000).kind, ADMIT) + + +def test_data_read_cap() raises: + var t = ClientTable() + assert_equal(admit(t, "a", True, 0, 2, 1000).kind, ADMIT) + # One at a time for a client. + var v = admit(t, "a", True, 0, 2, 1000) + assert_equal(v.kind, BUSY) + assert_equal(v.retry_after, 5) + # Other pages still go. + assert_equal(admit(t, "a", False, 0, 2, 1000).kind, ADMIT) + assert_equal(admit(t, "b", True, 0, 2, 1000).kind, ADMIT) + # Two running: the cap. + assert_equal(admit(t, "c", True, 0, 2, 1000).kind, BUSY) + release(t, "a") + assert_equal(admit(t, "c", True, 0, 2, 1000).kind, ADMIT) + assert_equal(admit(t, "a", True, 0, 2, 1000).kind, BUSY) + + +def test_stale_clients_dropped() raises: + var t = ClientTable() + for i in range(5000): + _ = admit(t, String("c", i), False, 0, 2, 1000) + assert_equal(len(t.clients), 5000) + # At 4096 the prune found nothing stale (the same minute), so the next + # is at 8192, and drops the 5000 whose minute has ended. + for i in range(5000, 8192): + _ = admit(t, String("c", i), False, 0, 2, 1100) + _ = admit(t, "late", False, 0, 2, 1100) + assert_equal(len(t.clients), 3192 + 1) + + +def main() raises: + TestSuite.discover_tests[__functions_in_module()]().run() diff --git a/tests/ui_drain.sh b/tests/ui_drain.sh index 6612a1f5..acd9d60b 100755 --- a/tests/ui_drain.sh +++ b/tests/ui_drain.sh @@ -11,7 +11,8 @@ # - SIGINT with a request that outlasts MORCA_UI_DRAIN_SECONDS: it exits # (1, the request cut) within the bound plus half a second; # - on 2 workers (MORCA_UI_WORKERS), a new request is answered while a slow -# one waits, both are logged, and SIGTERM still lets the slow one finish. +# one waits, both are logged, and SIGTERM still lets the slow one finish; +# - a client over MORCA_UI_RATE gets 429, unless it has a session cookie. # # The first three run on one worker: with several, each worker's connections # keep being accepted until the drain stops the loop. @@ -162,7 +163,7 @@ locker= wait "$slow_curl" 2> /dev/null || true # Two workers: a request gets its answer while the other waits on the lock. start 5 2 -grep -q "morca ui on http://127.0.0.1:$port (2 workers)" "$log" || fail "no 2 workers message" +grep -q "morca ui on http://127.0.0.1:$port (2 workers, 1 data reads at once)" "$log" || fail "no 2 workers message" lock 3 slow fast=$(curl -s -o /dev/null -w '%{http_code} %{time_total}' --max-time 2 "$base/static/morca.css" 2> /dev/null || true) @@ -174,5 +175,22 @@ wait "$slow_curl" || true [ "$(cat build/ui-drain-slow.txt)" = 200 ] || fail "the slow request on 2 workers got $(cat build/ui-drain-slow.txt)" wait "$locker" || true locker= -grep -Eq '^morca ui: GET /runs 200 [0-9]+ ms 127\.0\.0\.1 "curl/' "$log" || fail "/runs wasn't logged" +grep -Eq '^morca ui: [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}Z GET /runs 200 [0-9]+ ms 127\.0\.0\.1 "curl/' "$log" || fail "/runs wasn't logged" +# A client's rate (MORCA_UI_RATE, morca.web.limit): start's request and two +# more go, the fourth in the minute is 429 with Retry-After, and one with a +# session cookie isn't counted. +MORCA_UI_RATE=3 start 1 +grep -q "3 requests a minute a client)" "$log" || fail "no rate in the start message" +for i in 1 2; do + got=$(curl -s -o /dev/null -w '%{http_code}' "$base/runs") + [ "$got" = 200 ] || fail "request $i under the rate got $got" +done +got=$(curl -s -D build/ui-drain-429.h -o /dev/null -w '%{http_code}' "$base/runs") +[ "$got" = 429 ] || fail "the request over the rate got $got" +grep -qi '^retry-after: [0-9]' build/ui-drain-429.h || fail "no Retry-After on the 429" +got=$(curl -s -o /dev/null -w '%{http_code}' -H 'Cookie: morca_session=x' "$base/runs") +[ "$got" != 429 ] || fail "a request with a session cookie was counted" +grep -Eq '^morca ui: [0-9TZ:-]+ GET /runs 429 ' "$log" || fail "the 429 wasn't logged" +kill -TERM "$ui" +stopped 30 echo "ui drain: ok"