Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/547-ui-turnstile.minor.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **ui:** with `MORCA_UI_TURNSTILE_*` set, a visitor who is not logged in passes Cloudflare Turnstile once a day before a data read; Hetzner deploy tools 0.37.0 set it from `TURNSTILE_SITEKEY` and `secrets/orca/turnstile-secret`.
2 changes: 1 addition & 1 deletion deploy/hetzner/VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.36.0
0.37.0
9 changes: 9 additions & 0 deletions deploy/hetzner/hetzner.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,15 @@ UI_URL=
#HCLOUD_POOL_TOKEN_FILE=
#POOL_FIREWALL=
#POOL_SSH_KEYS=
# Cloudflare Turnstile before the UI's data reads (morca ui): a visitor who
# isn't logged in passes it once a day. Its site key (public) and its secret
# key: the key secrets/orca/turnstile-secret in SECRETS_FILE, or else
# TURNSTILE_SECRET_FILE, a file with it (or a .sops one). The hostnames a
# pass may come from, comma separated, are UI_URL's unless set. Off without
# both the site key and the secret.
#TURNSTILE_SITEKEY=
#TURNSTILE_SECRET_FILE=
#TURNSTILE_HOSTNAMES=

# Each run gets a cgroup with a memory limit at its grant (docs/hetzner.md,
# A cgroup per run). That's the server's setting, not this file's: on by
Expand Down
21 changes: 18 additions & 3 deletions deploy/hetzner/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ die() {
# The settings that name a file: a relative one is read from the folder of
# the settings file that sets it (resolve_paths).
PATH_SETTINGS="SECRETS_FILE S3_CREDENTIALS_FILE TUNNEL_TOKEN_FILE GITHUB_TOKEN_FILE
NTFY_TOPIC_FILE NTFY_TOKEN_FILE GHCR_TOKEN_FILE EXTRA_ENV_FILE CONFIG_OVERLAY_FILE SSH_PUBLIC_KEY_FILE SSH_PRIVATE_KEY_FILE"
NTFY_TOPIC_FILE NTFY_TOKEN_FILE GHCR_TOKEN_FILE TURNSTILE_SECRET_FILE EXTRA_ENV_FILE CONFIG_OVERLAY_FILE SSH_PUBLIC_KEY_FILE SSH_PRIVATE_KEY_FILE"

# resolve_paths DIR: each PATH_SETTINGS that is set and relative becomes
# DIR/<it>. Run after each settings file, so a shared hetzner.env (in a
Expand Down Expand Up @@ -445,7 +445,7 @@ need_settings() {
done
[ -z "${GHCR_USER:-}" ] || has_secret ghcr-token \
|| die "GHCR_USER needs GHCR_TOKEN_FILE or a ghcr-token key in SECRETS_FILE"
for v in S3_CREDENTIALS_FILE TUNNEL_TOKEN_FILE GITHUB_TOKEN_FILE NTFY_TOPIC_FILE NTFY_TOKEN_FILE GHCR_TOKEN_FILE CONFIG_OVERLAY_FILE; do
for v in S3_CREDENTIALS_FILE TUNNEL_TOKEN_FILE GITHUB_TOKEN_FILE NTFY_TOPIC_FILE NTFY_TOKEN_FILE GHCR_TOKEN_FILE TURNSTILE_SECRET_FILE CONFIG_OVERLAY_FILE; do
eval "val=\${$v:-}"
if is_sops "$val"; then
need_tool sops
Expand Down Expand Up @@ -485,7 +485,7 @@ need_handover_wait() {
# value is the file's content (a string). It may have more secrets/orca/<id>
# keys, the secrets the apps' [apps.<name>] secrets name (#331), sent the same way. One decrypt per run: a Secure
# Enclave age key (age-plugin-se) asks for Touch ID once, not per file.
SECRET_NAMES="s3.env tunnel.env secrets/orca/github-token secrets/orca/ntfy-topic secrets/orca/ntfy-token ghcr-token secrets/orca/hcloud-token"
SECRET_NAMES="s3.env tunnel.env secrets/orca/github-token secrets/orca/ntfy-topic secrets/orca/ntfy-token ghcr-token secrets/orca/hcloud-token secrets/orca/turnstile-secret"
SECRETS_JSON=
SECRETS_KEYS=
SECRETS_LOADED=0
Expand All @@ -500,6 +500,7 @@ secret_var() {
secrets/orca/ntfy-token) echo NTFY_TOKEN_FILE ;;
ghcr-token) echo GHCR_TOKEN_FILE ;;
secrets/orca/hcloud-token) echo HCLOUD_POOL_TOKEN_FILE ;;
secrets/orca/turnstile-secret) echo TURNSTILE_SECRET_FILE ;;
esac
}

Expand Down Expand Up @@ -603,6 +604,20 @@ write_settings() {
fi
# Where links point, and where pool machines reach the UI.
[ -z "${UI_URL:-}" ] || echo "MORCA_UI_URL=$UI_URL"
# Cloudflare Turnstile before the UI's data reads: the site key is
# public, the secret a secret file whose id is here; the hostnames a
# pass may come from are UI_URL's unless TURNSTILE_HOSTNAMES says.
if [ -n "${TURNSTILE_SITEKEY:-}" ] && has_secret secrets/orca/turnstile-secret; then
echo "MORCA_UI_TURNSTILE_SITEKEY=$TURNSTILE_SITEKEY"
echo "MORCA_UI_TURNSTILE_SECRET=orca/turnstile-secret"
ts_hosts=${TURNSTILE_HOSTNAMES:-}
if [ -z "$ts_hosts" ] && [ -n "${UI_URL:-}" ]; then
ts_hosts=${UI_URL#*://}
ts_hosts=${ts_hosts%%/*}
ts_hosts=${ts_hosts%%:*}
fi
[ -z "$ts_hosts" ] || echo "MORCA_UI_TURNSTILE_HOSTNAMES=$ts_hosts"
fi
# Worker pools morca grows itself ([pools.<name>], morca #267): the
# Hetzner token is a secret file, only its id is here; the machines
# get the stack's firewall and SSH key unless POOL_FIREWALL and
Expand Down
2 changes: 1 addition & 1 deletion deploy/hetzner/server/morca-stack
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ cmd_configure() {

# The files sync may write (in $CONF): settings.env and the secret files
# up.sh and deploy.sh send.
SYNC_FILES="settings.env overlay/morca.overlay.toml s3.env tunnel.env ghcr-token secrets/orca/github-token secrets/orca/ntfy-topic secrets/orca/ntfy-token secrets/orca/hcloud-token"
SYNC_FILES="settings.env overlay/morca.overlay.toml s3.env tunnel.env ghcr-token secrets/orca/github-token secrets/orca/ntfy-topic secrets/orca/ntfy-token secrets/orca/hcloud-token secrets/orca/turnstile-secret"

# key_changes OLD NEW: "added A, B; removed C; changed D" between two
# KEY=VALUE files (the last line of a key counts, as docker's --env-file
Expand Down
1 change: 1 addition & 0 deletions docs/hetzner.md
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ secrets/orca/github-token: ... # GITHUB_TOKEN_FILE's
secrets/orca/ntfy-topic: ... # NTFY_TOPIC_FILE's
secrets/orca/ntfy-token: ... # NTFY_TOKEN_FILE's
ghcr-token: ... # GHCR_TOKEN_FILE's
secrets/orca/turnstile-secret: ... # TURNSTILE_SECRET_FILE's
```

Write it with `sops edit prod/secrets.sops.yaml`, or, from the repository's
Expand Down
24 changes: 24 additions & 0 deletions docs/ui.md
Original file line number Diff line number Diff line change
Expand Up @@ -750,6 +750,30 @@ are `/healthz`, the stylesheet, `/robots.txt` and the worker API.
`X-Robots-Tag: noindex, nofollow`: run, snapshot and data pages are endless
and the data views are costly.

## Turnstile

Crawlers that come from a new address each time get past the limits per
client. With Cloudflare Turnstile on, a visitor who isn't logged in passes
a check once a day before a data read (a Data tab, `data.csv`,
`data.json`): the page shows the widget, which posts its token to
`POST /turnstile`; the UI asks Cloudflare's siteverify, needs `success`,
the action `data` and one of the hostnames, then sets the `morca_human`
cookie (a day; signed with a key the UI makes at start, so a restart asks
again) and sends the browser back. Without the cookie, `data.csv`,
`data.json` and JSON get a 403 in text. People logged in, API tokens and
the worker API never see it; nothing else is checked.

| Variable | |
|---|---|
| `MORCA_UI_TURNSTILE_SITEKEY` | the widget's site key (public) |
| `MORCA_UI_TURNSTILE_SECRET` | the id of its secret key, read as the other secrets are |
| `MORCA_UI_TURNSTILE_HOSTNAMES` | the hostnames a pass may come from, comma separated |

It's off unless the site key and the secret are both set. On Hetzner,
`TURNSTILE_SITEKEY`, `secrets/orca/turnstile-secret` in `SECRETS_FILE` (or
`TURNSTILE_SECRET_FILE`) and `TURNSTILE_HOSTNAMES` (UI_URL's host unless
set) in `hetzner.env` set them.

## Stopping

SIGTERM (what `docker stop` sends) or SIGINT (Ctrl-C) drains `morca ui`
Expand Down
3 changes: 2 additions & 1 deletion pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,7 @@ test-weekly = { cmd = "scripts/with-postgres.sh tests/weekly.sh" }
# the file and records, config show and export, and the env on each runner.
test-toml = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-toml" }
test-limit = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-limit" }
test-turnstile = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-turnstile" }
test-upgrade = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-upgrade" }
test-config = { cmd = "scripts/unit-tests.sh build scheduler-ui && scripts/unit-tests.sh run test-config && scripts/with-postgres.sh tests/config.sh" }
# Fills (morca.fill): a pipeline filling its range by itself, within limits.
Expand Down Expand Up @@ -300,7 +301,7 @@ test-executor-core = { depends-on = ["build-shared", "test-sample", "test-steps"
test-executor-workers = { depends-on = ["build-shared", "test-ec2", "test-autoscale", "test-no-workers", "test-pool", "test-hcloud", "test-pool-scale", "test-pool-scale-ec2", "test-pool-image", "test-run-cluster", "test-secrets", "test-placement", "test-grants", "test-pressure", "test-lost-workers"] }
test-executor = { depends-on = ["test-executor-core", "test-executor-workers"] }
# The scheduler, the UI, and what a registration and its code control:
test-scheduler-ui = { depends-on = ["build-shared", "test-toml", "test-upgrade", "test-config", "test-fill", "test-fill-version", "test-fill-requests", "test-priority", "test-pool-fills", "test-run-locks", "test-metrics", "test-terms", "test-scheduler", "test-ui", "test-fleet", "test-system", "test-actions", "test-calendar", "test-ui-login", "test-ui-drain", "test-remote", "test-worker-api", "test-pause", "test-older-binary", "test-watchdog", "test-limit", "test-notify", "test-versions", "test-manual-run", "test-new-code", "test-defer", "test-dead-runs", "test-adopt", "test-overlap", "test-outdated", "test-concurrency", "test-one-run-per-key", "test-hourly", "test-weekly", "test-sense-errors", "test-stop", "test-binary-timeouts"] }
test-scheduler-ui = { depends-on = ["build-shared", "test-toml", "test-upgrade", "test-config", "test-fill", "test-fill-version", "test-fill-requests", "test-priority", "test-pool-fills", "test-run-locks", "test-metrics", "test-terms", "test-scheduler", "test-ui", "test-fleet", "test-system", "test-actions", "test-calendar", "test-ui-login", "test-ui-drain", "test-remote", "test-worker-api", "test-pause", "test-older-binary", "test-watchdog", "test-limit", "test-turnstile", "test-notify", "test-versions", "test-manual-run", "test-new-code", "test-defer", "test-dead-runs", "test-adopt", "test-overlap", "test-outdated", "test-concurrency", "test-one-run-per-key", "test-hourly", "test-weekly", "test-sense-errors", "test-stop", "test-binary-timeouts"] }
test = { depends-on = ["test-executor", "test-scheduler-ui"] }
toy = { cmd = "scripts/mojo-build.sh -I src $DEP_INC examples/toy.mojo -o build/toy" }
fanout = { cmd = "scripts/mojo-build.sh -I src $DEP_INC examples/fanout.mojo -o build/fanout" }
Expand Down
1 change: 1 addition & 0 deletions scripts/unit-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ scheduler-ui:tests/test_remote.mojo:test-remote
scheduler-ui:tests/test_worker.mojo:test-worker-api
scheduler-ui:tests/test_watchdog.mojo:test-watchdog
scheduler-ui:tests/test_limit.mojo:test-limit
scheduler-ui:tests/test_turnstile.mojo:test-turnstile
data-tables:tests/test_tables.mojo:test-tables
data-tables:tests/test_lake_batch.mojo:test-lake-batch
data-tables:tests/test_replace.mojo:test-replace
Expand Down
44 changes: 40 additions & 4 deletions src/morca/_internal/web/server.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,10 @@ A client may send `MORCA_UI_RATE` requests a minute (off unless set; the
Hetzner stack sets 120), and the data reads, which take a second or more,
run `MORCA_UI_DATA_READS` at once (half the workers), one per client, so
the other pages don't wait behind them; people logged in and scripts aren't
limited (`morca._internal.web.limit`).
limited (`morca._internal.web.limit`). With `MORCA_UI_TURNSTILE_SITEKEY` and
`MORCA_UI_TURNSTILE_SECRET` set, a visitor who isn't logged in passes
Cloudflare Turnstile once a day before a data read
(`morca._internal.web.turnstile`, #547).
Each request but `/healthz` and the stylesheet is logged as one line: the
time (UTC, to the second), method, path, status, milliseconds, the client (`client`), Cloudflare's
`CF-Connecting-IP` when there is one, and the User-Agent.
Expand Down Expand Up @@ -89,6 +92,11 @@ from morca._internal.web.auth import (
)
from morca._internal.web.drain import Drain
from morca._internal.web.limit import Gate, Limits
from morca._internal.web.turnstile import (
VERIFY_PATH,
Turnstile,
challenge_page,
)
from morca._internal.web.page import Page, respond
from morca._internal.web.router import (
ACTION,
Expand Down Expand Up @@ -252,12 +260,21 @@ struct Ui(Copyable, Handler):
"""Set once a drain has started (`morca._internal.web.drain.Drain.draining`)."""
var gate: Gate
"""The limits per client (`morca._internal.web.limit`)."""

def __init__(out self, url: String, draining: AtomicFlag, gate: Gate):
var turnstile: Turnstile
"""The check before a data read (`morca._internal.web.turnstile`)."""

def __init__(
out self,
url: String,
draining: AtomicFlag,
gate: Gate,
var turnstile: Turnstile,
):
self.url = url
self.table = routes()
self.draining = draining
self.gate = gate
self.turnstile = turnstile^

def serve(self, req: Request) raises -> Response:
var path = _path_of(req.url)
Expand All @@ -282,6 +299,21 @@ struct Ui(Copyable, Handler):
return _healthz(self.url, self.draining.is_set())
if post and path.startswith("/api/worker/"):
return worker_api.worker_response(self.url, req)
if self.turnstile.on():
if post and path == VERIFY_PATH:
if not same_origin(req):
return respond(cross_origin(), False)
return self.turnstile.verify(req)
if self.turnstile.needs_page(req, path):
return _with_user(
challenge_page(self.turnstile.sitekey, req.url),
False,
"",
False,
)
var refused = self.turnstile.check(req, path)
if refused:
return refused.take()
var found = find(self.table, req.method, path)
var reading = req.method == "GET" or req.method == "HEAD"
if not found.route and not (reading or (post and found.miss)):
Expand Down Expand Up @@ -479,6 +511,7 @@ def serve_ui(url: String, addr: String, worker_addr: String = "") raises:
# that's busy with a slow page.
_ = setenv("FLARE_REUSEPORT_WORKERS", "0")
var limits = Limits(workers)
var turnstile = Turnstile()
print(
"morca ui on http://",
addr,
Expand All @@ -490,11 +523,14 @@ def serve_ui(url: String, addr: String, worker_addr: String = "") raises:
String(
", ", limits.rate, " requests a minute a client"
) if limits.rate else String(),
", Turnstile before data reads" if turnstile.on() else "",
")",
sep="",
)
server.serve(
Logged(drain.counting(Ui(url, drain.draining(), limits.gate()))),
Logged(
drain.counting(Ui(url, drain.draining(), limits.gate(), turnstile^))
),
workers,
)
_ = drain^
Expand Down
Loading
Loading