Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/542-ui-robots.minor.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **No crawlers**: the UI serves a `robots.txt` that disallows everything, and marks each page `noindex`. (#542)
4 changes: 4 additions & 0 deletions docs/ui.md
Original file line number Diff line number Diff line change
Expand Up @@ -729,6 +729,10 @@ That's the method, URL, status, milliseconds, and the client address (from
`CF-Connecting-IP` when the request came through a tunnel, and the
User-Agent.

`/robots.txt` disallows every crawler, and each answer carries
`X-Robots-Tag: noindex, nofollow`: run, snapshot and data pages are endless
and the data views are costly.

## Stopping

SIGTERM (what `docker stop` sends) or SIGINT (Ctrl-C) drains `morca ui`
Expand Down
23 changes: 22 additions & 1 deletion src/morca/_internal/web/server.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ from morca._internal.web.routes import routes
comptime DEFAULT_ADDR = "127.0.0.1:8270"
comptime STYLESHEET = "/static/morca.css"
comptime HEALTHZ = "/healthz"
comptime ROBOTS = "/robots.txt"


def _path_of(url: String) -> String:
Expand Down Expand Up @@ -188,6 +189,19 @@ def _stylesheet() raises -> Response:
return resp^


def _robots() raises -> Response:
"""`/robots.txt`: no crawler is welcome. Pages that a crawler can reach
without a login (runs, snapshots, data downloads) are endless, and the
data views cost the database a second each."""
var resp = Response(
status=200,
reason="OK",
body=List[UInt8]("User-agent: *\nDisallow: /\n".as_bytes()),
)
resp.headers.set("Content-Type", "text/plain; charset=utf-8")
return resp^


def _healthz(url: String, draining: Bool) -> Response:
"""`/healthz`: 200 `ok` once the database answers a query, 503 if it
doesn't, and 503 `draining` without asking it once SIGTERM or SIGINT
Expand Down Expand Up @@ -240,6 +254,8 @@ struct Ui(Copyable, Handler):
var post = req.method == "POST"
if path == STYLESHEET:
return _stylesheet()
if path == ROBOTS:
return _robots()
if path == HEALTHZ and (req.method == "GET" or req.method == "HEAD"):
return _healthz(self.url, self.draining.is_set())
if post and path.startswith("/api/worker/"):
Expand Down Expand Up @@ -353,7 +369,8 @@ def _log(req: Request, status: Int, start_ns: Int):

struct Logged[H: Handler & Copyable](Copyable, Handler):
"""`H`, logging each request but `/healthz` and the stylesheet as it's
answered."""
answered, and telling crawlers not to index any answer (`X-Robots-Tag`;
`/robots.txt` turns them away first)."""

comptime BodyType = Self.H.BodyType

Expand All @@ -368,6 +385,10 @@ struct Logged[H: Handler & Copyable](Copyable, Handler):
var quiet = path == HEALTHZ or path == STYLESHEET
try:
var resp = self.inner.serve(req)
try:
resp.headers.set("X-Robots-Tag", "noindex, nofollow")
except:
pass
if not quiet:
_log(req, resp.status, start)
return resp^
Expand Down
3 changes: 3 additions & 0 deletions tests/ui.sh
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,9 @@ done
code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "$base/runs")
[ "$code" = 405 ] || fail "POST answered $code, not 405"
curl -fsS "$base/static/morca.css" | grep -q -- '--ember' || fail "the stylesheet"
# No crawler: robots.txt turns them all away, and no page is to be indexed.
[ "$(curl -fsS "$base/robots.txt")" = "$(printf 'User-agent: *\nDisallow: /')" ] || fail "robots.txt"
curl -fsSI "$base/runs" | grep -qi '^X-Robots-Tag: noindex, nofollow' || fail "no X-Robots-Tag on /runs"

# An attempt's page: its error, its log, the step's other attempts.
failed=$(json "/runs/$bad_run" | jq -r '[.steps[] | select(.status == "failed")][0].attempt_id')
Expand Down
Loading