From f230e679cc43c6a98cdf78942ce8f98063280a63 Mon Sep 17 00:00:00 2001 From: Marius S <39998+winding-lines@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:38:02 -0800 Subject: [PATCH] morca ui: robots.txt disallows every crawler, and pages say noindex Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017TycszSbyw92Efu4ZeCYL6 --- changes/542-ui-robots.minor.md | 1 + docs/ui.md | 4 ++++ src/morca/_internal/web/server.mojo | 23 ++++++++++++++++++++++- tests/ui.sh | 3 +++ 4 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 changes/542-ui-robots.minor.md diff --git a/changes/542-ui-robots.minor.md b/changes/542-ui-robots.minor.md new file mode 100644 index 00000000..a70c79c1 --- /dev/null +++ b/changes/542-ui-robots.minor.md @@ -0,0 +1 @@ +- **No crawlers**: the UI serves a `robots.txt` that disallows everything, and marks each page `noindex`. (#542) diff --git a/docs/ui.md b/docs/ui.md index 696e9bcd..84413c4f 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -729,6 +729,10 @@ That's the method, URL, status, milliseconds, and the client address (from `CF-Connecting-IP` when the request came through a tunnel, and the User-Agent. +`/robots.txt` disallows every crawler, and each answer carries +`X-Robots-Tag: noindex, nofollow`: run, snapshot and data pages are endless +and the data views are costly. + ## Stopping SIGTERM (what `docker stop` sends) or SIGINT (Ctrl-C) drains `morca ui` diff --git a/src/morca/_internal/web/server.mojo b/src/morca/_internal/web/server.mojo index fa1347b5..37bbcd2b 100644 --- a/src/morca/_internal/web/server.mojo +++ b/src/morca/_internal/web/server.mojo @@ -97,6 +97,7 @@ from morca._internal.web.routes import routes comptime DEFAULT_ADDR = "127.0.0.1:8270" comptime STYLESHEET = "/static/morca.css" comptime HEALTHZ = "/healthz" +comptime ROBOTS = "/robots.txt" def _path_of(url: String) -> String: @@ -188,6 +189,19 @@ def _stylesheet() raises -> Response: return resp^ +def _robots() raises -> Response: + """`/robots.txt`: no crawler is welcome. Pages that a crawler can reach + without a login (runs, snapshots, data downloads) are endless, and the + data views cost the database a second each.""" + var resp = Response( + status=200, + reason="OK", + body=List[UInt8]("User-agent: *\nDisallow: /\n".as_bytes()), + ) + resp.headers.set("Content-Type", "text/plain; charset=utf-8") + return resp^ + + def _healthz(url: String, draining: Bool) -> Response: """`/healthz`: 200 `ok` once the database answers a query, 503 if it doesn't, and 503 `draining` without asking it once SIGTERM or SIGINT @@ -240,6 +254,8 @@ struct Ui(Copyable, Handler): var post = req.method == "POST" if path == STYLESHEET: return _stylesheet() + if path == ROBOTS: + return _robots() if path == HEALTHZ and (req.method == "GET" or req.method == "HEAD"): return _healthz(self.url, self.draining.is_set()) if post and path.startswith("/api/worker/"): @@ -353,7 +369,8 @@ def _log(req: Request, status: Int, start_ns: Int): struct Logged[H: Handler & Copyable](Copyable, Handler): """`H`, logging each request but `/healthz` and the stylesheet as it's - answered.""" + answered, and telling crawlers not to index any answer (`X-Robots-Tag`; + `/robots.txt` turns them away first).""" comptime BodyType = Self.H.BodyType @@ -368,6 +385,10 @@ struct Logged[H: Handler & Copyable](Copyable, Handler): var quiet = path == HEALTHZ or path == STYLESHEET try: var resp = self.inner.serve(req) + try: + resp.headers.set("X-Robots-Tag", "noindex, nofollow") + except: + pass if not quiet: _log(req, resp.status, start) return resp^ diff --git a/tests/ui.sh b/tests/ui.sh index 72dbeec4..638a7785 100755 --- a/tests/ui.sh +++ b/tests/ui.sh @@ -168,6 +168,9 @@ done code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "$base/runs") [ "$code" = 405 ] || fail "POST answered $code, not 405" curl -fsS "$base/static/morca.css" | grep -q -- '--ember' || fail "the stylesheet" +# No crawler: robots.txt turns them all away, and no page is to be indexed. +[ "$(curl -fsS "$base/robots.txt")" = "$(printf 'User-agent: *\nDisallow: /')" ] || fail "robots.txt" +curl -fsSI "$base/runs" | grep -qi '^X-Robots-Tag: noindex, nofollow' || fail "no X-Robots-Tag on /runs" # An attempt's page: its error, its log, the step's other attempts. failed=$(json "/runs/$bad_run" | jq -r '[.steps[] | select(.status == "failed")][0].attempt_id')