From e77a7f4c0ee9e861e45fdf45b4073099323485dd Mon Sep 17 00:00:00 2001 From: davidamacey Date: Fri, 9 Oct 2026 05:32:10 -0400 Subject: [PATCH 1/4] feat(links): serve path-relative service links in gateway mode --- src/config/curation.py | 6 ++ src/services/resource_links.py | 40 ++++++++--- tests/curation/test_resource_links.py | 96 +++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 8 deletions(-) diff --git a/src/config/curation.py b/src/config/curation.py index 50666c2f4..4b042b731 100644 --- a/src/config/curation.py +++ b/src/config/curation.py @@ -218,6 +218,11 @@ class CurationConfig: prometheus_port: int = 4604 dashboards_port: int = 4608 mlflow_port: int = 4609 + # Gateway mode (``OP_GATEWAY_SUBPATHS``): the frontend's nginx serves the + # four UIs under /grafana/, /prometheus/, /dashboards/ and /mlflow/ on its + # one published port, so their unset URLs are served path-relative instead + # of : (``src.services.resource_links.service_url``). + gateway_subpaths: bool = False # Confidence floor gating the item wire's `probe_actionable` field # (see `src.services.curation.wire.serialize_item` and @@ -391,6 +396,7 @@ def _bool(name: str, default: bool) -> bool: prometheus_port=_int('PROMETHEUS_PORT', defaults.prometheus_port), dashboards_port=_int('DASHBOARDS_PORT', defaults.dashboards_port), mlflow_port=_int('MLFLOW_PORT', defaults.mlflow_port), + gateway_subpaths=_bool('GATEWAY_SUBPATHS', defaults.gateway_subpaths), embedding_dim=_int('EMBEDDING_DIM', defaults.embedding_dim), encoder_embedding_dim=_int('ENCODER_EMBEDDING_DIM', defaults.encoder_embedding_dim), backbone_embedding_dim=_int('BACKBONE_EMBEDDING_DIM', defaults.backbone_embedding_dim), diff --git a/src/services/resource_links.py b/src/services/resource_links.py index 7217440ba..b73767ac0 100644 --- a/src/services/resource_links.py +++ b/src/services/resource_links.py @@ -3,11 +3,15 @@ URL rule (one function, :func:`service_url`, shared with the MLflow run link): a ``kind: 'service'`` URL is (1) the explicit ``OP__URL`` verbatim -when set (reverse-proxied deployments); else (2) ``://:`` from the request (``Host`` / +when set (reverse-proxied deployments); else ``null`` when the service's +``OP__PORT`` is 0 (disabled); else (2) with ``OP_GATEWAY_SUBPATHS`` the +gateway sub-path (``/grafana/``, ``/prometheus/``, ``/dashboards/``, +``/mlflow/``), which needs no request context because the browser resolves it +against the origin it already uses; else (3) ``://:`` from the request (``Host`` / ``X-Forwarded-Host`` / ``X-Forwarded-Proto``, see :mod:`src.core.request_origin`), so a LAN client gets its own host, never -``localhost``; else ``null`` (no request context, or the port is 0). A +``localhost``; else ``null`` (no request context). A ``kind: 'docs'`` URL is PATH-RELATIVE to the API origin (``/docs``). ``reachable`` means "the service answers FROM THE SERVER" (probed at its @@ -47,6 +51,7 @@ class _Service(NamedTuple): name: str # CurationConfig attribute stem: _url / _port env: str # OP__URL / OP__PORT probe_url: str # compose-internal address + gateway_path: str # sub-path the frontend nginx serves it under (gateway mode) _DOCS = ( @@ -57,9 +62,16 @@ class _Service(NamedTuple): # Probe URLs are the compose service names/container ports. _SERVICES = ( - _Service('grafana', 'Grafana', 'grafana', 'GRAFANA', 'http://grafana:3000/api/health'), _Service( - 'prometheus', 'Prometheus', 'prometheus', 'PROMETHEUS', 'http://prometheus:9090/-/healthy' + 'grafana', 'Grafana', 'grafana', 'GRAFANA', 'http://grafana:3000/api/health', '/grafana/' + ), + _Service( + 'prometheus', + 'Prometheus', + 'prometheus', + 'PROMETHEUS', + 'http://prometheus:9090/-/healthy', + '/prometheus/', ), _Service( 'opensearch_dashboards', @@ -67,8 +79,11 @@ class _Service(NamedTuple): 'dashboards', 'DASHBOARDS', 'http://opensearch-dashboards:5601/api/status', + '/dashboards/', + ), + _Service( + 'mlflow', 'MLflow', 'mlflow', 'MLFLOW', 'http://curation-mlflow:5000/health', '/mlflow/' ), - _Service('mlflow', 'MLflow', 'mlflow', 'MLFLOW', 'http://curation-mlflow:5000/health'), ) @@ -116,8 +131,12 @@ def service_url(service_id: str) -> str | None: if explicit: return explicit port = getattr(cfg, f'{spec.name}_port') + if not port: + return None + if cfg.gateway_subpaths: + return spec.gateway_path origin = current_origin() - if not port or origin is None: + if origin is None: return None return f'{origin.scheme}://{origin.host}:{port}' @@ -152,6 +171,9 @@ async def wait_for_inflight_refresh() -> None: def resource_links_from_config() -> list[ResourceLink]: + from src.config import get_curation_config + + gateway = get_curation_config().gateway_subpaths links = [ ResourceLink(id=id_, label=label, url=path, kind='docs', status='configured', hint=hint) for id_, label, path, hint in _DOCS @@ -166,7 +188,9 @@ def resource_links_from_config() -> list[ResourceLink]: kind='service', status='configured' if url else 'not_configured', hint=( - f'Set {_url_env(s)} to override; default is this host on OP_{s.env}_PORT.' + f'Set {_url_env(s)} to override; served by the gateway at {s.gateway_path}.' + if url == s.gateway_path and gateway + else f'Set {_url_env(s)} to override; default is this host on OP_{s.env}_PORT.' if url else f'Set OP_{s.env}_PORT (0 disables) or {_url_env(s)}.' ), diff --git a/tests/curation/test_resource_links.py b/tests/curation/test_resource_links.py index 60a1253e2..3cfcd4fe8 100644 --- a/tests/curation/test_resource_links.py +++ b/tests/curation/test_resource_links.py @@ -200,3 +200,99 @@ async def slow_probe(url: str) -> bool: @pytest.mark.asyncio async def test_probe_failure_is_false_not_raise() -> None: assert await rl._probe('http://127.0.0.1:1/x') is False + + +# --- gateway mode (OP_GATEWAY_SUBPATHS): path-relative links --------------- + +_GATEWAY_PATHS = { + 'grafana': '/grafana/', + 'prometheus': '/prometheus/', + 'opensearch_dashboards': '/dashboards/', + 'mlflow': '/mlflow/', +} + + +def test_gateway_mode_serves_path_relative_links(monkeypatch: pytest.MonkeyPatch) -> None: + _cfg(monkeypatch, gateway_subpaths=True) + body = _app().get('/links', headers={'Host': '10.10.10.20:5184'}).json() + assert {k: body[k] for k in _GATEWAY_PATHS} == _GATEWAY_PATHS + assert body['swagger'] == '/docs' + + +def test_gateway_mode_needs_no_request_context(monkeypatch: pytest.MonkeyPatch) -> None: + _cfg(monkeypatch, gateway_subpaths=True) + links = _by_id(rl.resource_links_from_config()) + for id_, path in _GATEWAY_PATHS.items(): + assert (links[id_].url, links[id_].status) == (path, 'configured') + assert rl.service_url('mlflow') == '/mlflow/' + + +def test_gateway_off_keeps_host_port_derivation(monkeypatch: pytest.MonkeyPatch) -> None: + _cfg(monkeypatch, gateway_subpaths=False) + assert _app().get('/links', headers={'Host': '10.10.10.20:5184'}).json()['grafana'] == ( + 'http://10.10.10.20:4605' + ) + assert rl.service_url('grafana') is None + + +def test_gateway_mode_explicit_override_still_wins(monkeypatch: pytest.MonkeyPatch) -> None: + _cfg( + monkeypatch, + gateway_subpaths=True, + grafana_url='https://grafana.example.com/g', + mlflow_public_url='http://m:4', + ) + links = _by_id(rl.resource_links_from_config()) + assert links['grafana'].url == 'https://grafana.example.com/g' + assert links['mlflow'].url == 'http://m:4' + assert links['prometheus'].url == '/prometheus/' + + +def test_gateway_mode_port_zero_still_disables(monkeypatch: pytest.MonkeyPatch) -> None: + _cfg(monkeypatch, gateway_subpaths=True, grafana_port=0) + links = _by_id(rl.resource_links_from_config()) + assert (links['grafana'].url, links['grafana'].status) == (None, 'not_configured') + assert links['mlflow'].url == '/mlflow/' + + +def test_gateway_mode_port_zero_with_explicit_url_is_enabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _cfg(monkeypatch, gateway_subpaths=True, mlflow_port=0, mlflow_public_url='http://m:4') + assert rl.service_url('mlflow') == 'http://m:4' + + +def test_gateway_mode_hint_names_the_gateway(monkeypatch: pytest.MonkeyPatch) -> None: + _cfg(monkeypatch, gateway_subpaths=True) + assert 'gateway' in _by_id(rl.resource_links_from_config())['grafana'].hint.lower() + + +@pytest.mark.asyncio +async def test_gateway_mode_still_probes_enabled_services_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _cfg(monkeypatch, gateway_subpaths=True, prometheus_port=0, dashboards_port=0) + seen: list[str] = [] + + async def fake_probe(url: str) -> bool: + seen.append(url) + return True + + monkeypatch.setattr(rl, '_probe', fake_probe) + await rl.refresh_reachability() + assert sorted(seen) == ['http://curation-mlflow:5000/health', 'http://grafana:3000/api/health'] + + +def test_mlflow_run_url_follows_the_gateway(monkeypatch: pytest.MonkeyPatch) -> None: + from src.services.training.jobs import _public_mlflow_url + + _cfg(monkeypatch, gateway_subpaths=True) + assert _public_mlflow_url('run1', '7') == '/mlflow/#/experiments/7/runs/run1' + + +def test_gateway_flag_is_read_from_the_environment(monkeypatch: pytest.MonkeyPatch) -> None: + assert CurationConfig.from_env().gateway_subpaths is False + monkeypatch.setenv('OP_GATEWAY_SUBPATHS', 'true') + assert CurationConfig.from_env().gateway_subpaths is True + monkeypatch.setenv('OP_GATEWAY_SUBPATHS', 'false') + assert CurationConfig.from_env().gateway_subpaths is False From 982c2a2c4b0dc815201b51dc653384488176423e Mon Sep 17 00:00:00 2001 From: davidamacey Date: Fri, 9 Oct 2026 05:32:10 -0400 Subject: [PATCH 2/4] feat(frontend): proxy the monitoring UIs under sub-paths in gateway mode --- frontend/CHANGELOG.md | 12 ++ frontend/CLAUDE.md | 10 ++ frontend/Dockerfile | 6 +- frontend/docker-entrypoint.sh | 39 +++++ frontend/nginx-gateway.conf | 100 +++++++++++++ frontend/nginx.conf | 10 ++ frontend/src/lib/entrypoint.test.ts | 3 +- frontend/src/lib/gatewayConf.test.ts | 216 +++++++++++++++++++++++++++ 8 files changed, 393 insertions(+), 3 deletions(-) create mode 100644 frontend/nginx-gateway.conf create mode 100644 frontend/src/lib/gatewayConf.test.ts diff --git a/frontend/CHANGELOG.md b/frontend/CHANGELOG.md index 154ca57f6..c6713eee3 100644 --- a/frontend/CHANGELOG.md +++ b/frontend/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Gateway mode: with `OP_GATEWAY_SUBPATHS=true` the entrypoint installs + `nginx-gateway.conf`, which proxies `/grafana/`, `/prometheus/`, `/dashboards/` and + `/mlflow/` to the compose services (bare service names only, overridable with + `GRAFANA_UPSTREAM`, `PROMETHEUS_UPSTREAM`, `DASHBOARDS_UPSTREAM`, `MLFLOW_UPSTREAM`), + with the websocket upgrade for Grafana Live. No ports are hardcoded in the app. + +### Fixed + +- nginx uses `absolute_redirect off`, so redirects no longer point at container port 8080. + ### Changed - Moved to SvelteKit 3 and `@sveltejs/adapter-static` 4 on TypeScript 6. SvelteKit diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index d704e37e3..2eef540d6 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -2712,6 +2712,16 @@ resets and re-reads on a project switch). Tests: `resourceLinks.test.ts`, `contract/curationSettingsContract.test.ts`, the nginx cases in `entrypoint.test.ts`, e2e `test_resources_menu.py`. +### Gateway mode (OpenProcessor #93) + +`OP_GATEWAY_SUBPATHS=true` (read by `docker-entrypoint.sh`, same spelling as the +API's) copies `nginx-gateway.conf` into `/etc/nginx/gateway.d/`, which `nginx.conf` +includes; off, the directory is empty. It proxies `/grafana/`, `/prometheus/`, +`/dashboards/` and `/mlflow/` (MLflow with `/mlflow` stripped and its own service +name as `Host`; Grafana Live upgraded) to bare compose service names, validated by +the entrypoint. The app needs no code for it: the served `resource_links` become +path-relative and `safeHref` already accepts them. Tests: `gatewayConf.test.ts`. + ### Projects — `/p/[project]` routes, switcher, `/projects` **OWNER DECISION: a fresh build, no backward compatibility.** The diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 1f19404c6..f43bb10cb 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -31,12 +31,14 @@ RUN apk upgrade --no-cache && apk del --no-cache curl && rm -rf /usr/share/nginx COPY --from=builder /app/build /usr/share/nginx/html COPY nginx.conf /etc/nginx/conf.d/default.conf COPY nginx-security-headers.conf /etc/nginx/snippets/security-headers.conf +COPY nginx-gateway.conf /etc/nginx/snippets/gateway-subpaths.conf COPY docker-entrypoint.sh /docker-entrypoint.d/40-runtime-config.sh # The runtime-config script rewrites placeholders in the bundle and in # default.conf with `sed -i` at container start, as uid 101. RUN chmod 0755 /docker-entrypoint.d/40-runtime-config.sh \ - && chown -R 101:0 /usr/share/nginx/html /etc/nginx/conf.d \ - && chmod -R g+w /usr/share/nginx/html /etc/nginx/conf.d + && mkdir -p /etc/nginx/gateway.d \ + && chown -R 101:0 /usr/share/nginx/html /etc/nginx/conf.d /etc/nginx/gateway.d \ + && chmod -R g+w /usr/share/nginx/html /etc/nginx/conf.d /etc/nginx/gateway.d USER 101 EXPOSE 8080 diff --git a/frontend/docker-entrypoint.sh b/frontend/docker-entrypoint.sh index 2b340a74e..5170e3af3 100755 --- a/frontend/docker-entrypoint.sh +++ b/frontend/docker-entrypoint.sh @@ -88,6 +88,44 @@ case "$DATASET_UPLOAD_MAX_MB" in ;; esac +# Gateway mode: serve the monitoring UIs under sub-paths of this one origin. +# Same spelling as the API's OP_GATEWAY_SUBPATHS (CurationConfig._bool), plus an +# explicit refusal of anything else so a typo cannot silently leave it off. +GATEWAY_DIR=/etc/nginx/gateway.d +GATEWAY_SNIPPET=/etc/nginx/snippets/gateway-subpaths.conf +GATEWAY_ON=0 +case "$(printf '%s' "${OP_GATEWAY_SUBPATHS:-}" | tr 'A-Z' 'a-z')" in + 1|true|yes|on) GATEWAY_ON=1 ;; + ''|0|false|no|off) ;; + *) + echo "[entrypoint] OP_GATEWAY_SUBPATHS must be true/false (1, true, yes, on / 0, false, no, off), got: ${OP_GATEWAY_SUBPATHS}" >&2 + exit 1 + ;; +esac + +# A UI upstream is a bare compose service name over plain http: no IP, no +# dotted name, no path. That is what keeps the gateway from proxying to +# anything outside the compose network. +gateway_upstream() { + var="$1"; value="$2" + if ! printf '%s' "$value" | grep -Eq '^http://[a-z][a-z0-9-]*:[0-9]{1,5}$'; then + echo "[entrypoint] $var must be http://: (single-label name, no IP or path), got: $value" >&2 + exit 1 + fi + printf '%s' "$value" +} + +mkdir -p "$GATEWAY_DIR" +rm -f "$GATEWAY_DIR/gateway-subpaths.conf" +if [ "$GATEWAY_ON" = 1 ]; then + GRAFANA_UPSTREAM="$(gateway_upstream GRAFANA_UPSTREAM "${GRAFANA_UPSTREAM:-http://grafana:3000}")" + PROMETHEUS_UPSTREAM="$(gateway_upstream PROMETHEUS_UPSTREAM "${PROMETHEUS_UPSTREAM:-http://prometheus:9090}")" + DASHBOARDS_UPSTREAM="$(gateway_upstream DASHBOARDS_UPSTREAM "${DASHBOARDS_UPSTREAM:-http://opensearch-dashboards:5601}")" + MLFLOW_UPSTREAM="$(gateway_upstream MLFLOW_UPSTREAM "${MLFLOW_UPSTREAM:-http://curation-mlflow:5000}")" + sed "s|__GRAFANA_UPSTREAM__|${GRAFANA_UPSTREAM}|g; s|__PROMETHEUS_UPSTREAM__|${PROMETHEUS_UPSTREAM}|g; s|__DASHBOARDS_UPSTREAM__|${DASHBOARDS_UPSTREAM}|g; s|__MLFLOW_UPSTREAM__|${MLFLOW_UPSTREAM}|g" \ + "$GATEWAY_SNIPPET" > "$GATEWAY_DIR/gateway-subpaths.conf" +fi + find /usr/share/nginx/html -type f \( -name '*.js' -o -name '*.html' \) \ -exec sed -i "s|__RUNTIME__|${TARGET_URL}|g; s|__API_PREFIX__|${API_PREFIX}|g; s|__INGEST_MAX_REQUEST_MB__|${INGEST_MAX_REQUEST_MB}|g; s|__DATASET_UPLOAD_MAX_MB__|${DATASET_UPLOAD_MAX_MB}|g" {} + @@ -102,3 +140,4 @@ echo "[entrypoint] API_UPSTREAM=${API_UPSTREAM}" echo "[entrypoint] DOCS_UPSTREAM=${DOCS_UPSTREAM}" echo "[entrypoint] CROPWRIGHT_INGEST_MAX_REQUEST_MB=${INGEST_MAX_REQUEST_MB}" echo "[entrypoint] CROPWRIGHT_DATASET_UPLOAD_MAX_MB=${DATASET_UPLOAD_MAX_MB}" +echo "[entrypoint] OP_GATEWAY_SUBPATHS=${GATEWAY_ON} (1 = monitoring UIs proxied under /grafana/, /prometheus/, /dashboards/, /mlflow/)" diff --git a/frontend/nginx-gateway.conf b/frontend/nginx-gateway.conf new file mode 100644 index 000000000..0b2f55f44 --- /dev/null +++ b/frontend/nginx-gateway.conf @@ -0,0 +1,100 @@ +# Gateway sub-path routes for the monitoring UIs. Installed into +# /etc/nginx/gateway.d/ by docker-entrypoint.sh ONLY when OP_GATEWAY_SUBPATHS is +# true; nginx.conf includes that directory inside the server block. +# +# Every upstream is a bare compose service name (validated by the entrypoint): +# these locations can never reach an address outside the compose network. +# Variable-form proxy_pass + the resolver in nginx.conf, as for the API, so a +# stopped or restarted UI only 502s its own sub-path. +# +# Grafana, Prometheus and OpenSearch Dashboards are configured to serve FROM +# their sub-path (OP_GATEWAY_SUBPATHS in docker-compose.yml), so the request URI +# is passed through unchanged. MLflow is left on its normal root (its UI uses +# relative URLs, and --static-prefix would also move /health and /api, breaking +# the trainer's and API's in-network MLFLOW_TRACKING_URI), so /mlflow is stripped +# before the upstream. +# +# Host and X-Forwarded-* carry the original client host, as on every API +# location; MLflow is the one exception for Host (see its location). + +set $grafana_upstream __GRAFANA_UPSTREAM__; +set $prometheus_upstream __PROMETHEUS_UPSTREAM__; +set $dashboards_upstream __DASHBOARDS_UPSTREAM__; +set $mlflow_upstream __MLFLOW_UPSTREAM__; + +location = /grafana { + return 301 /grafana/; +} +# Grafana Live: a long-lived websocket. +location ^~ /grafana/api/live/ { + proxy_pass $grafana_upstream; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 10s; + proxy_read_timeout 3600s; +} +location ^~ /grafana/ { + proxy_pass $grafana_upstream; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 10s; + proxy_read_timeout 120s; +} + +location = /prometheus { + return 301 /prometheus/; +} +location ^~ /prometheus/ { + proxy_pass $prometheus_upstream; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 10s; + proxy_read_timeout 120s; +} + +location = /dashboards { + return 301 /dashboards/; +} +location ^~ /dashboards/ { + proxy_pass $dashboards_upstream; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 10s; + proxy_read_timeout 300s; +} + +location = /mlflow { + return 301 /mlflow/; +} +# MLflow 3 answers 403 "Invalid Host header" to a Host outside +# MLFLOW_SERVER_ALLOWED_HOSTS. The gateway is the only caller, so it sends the +# upstream's own name (always on that list) instead of the browser's host, which +# would need every LAN name or address added; the browser's host still travels +# in X-Forwarded-Host. +location ^~ /mlflow/ { + rewrite ^/mlflow/(.*)$ /$1 break; + proxy_pass $mlflow_upstream; + proxy_set_header Host $proxy_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 10s; + proxy_read_timeout 120s; + client_max_body_size 2048m; +} diff --git a/frontend/nginx.conf b/frontend/nginx.conf index d8c5dc789..ebb181530 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -6,6 +6,11 @@ server { server_tokens off; + # nginx listens on 8080 inside the container; the published port differs. Without + # this a `return 301 /grafana/;` becomes http://:8080/grafana/, a port the + # browser cannot reach. + absolute_redirect off; + gzip on; gzip_types text/css application/javascript application/json image/svg+xml; gzip_min_length 256; @@ -191,6 +196,11 @@ server { proxy_read_timeout 60s; } + # Gateway sub-path routes for the monitoring UIs (/grafana/, /prometheus/, + # /dashboards/, /mlflow/): docker-entrypoint.sh installs nginx-gateway.conf + # here only when OP_GATEWAY_SUBPATHS is true; otherwise the directory is empty. + include /etc/nginx/gateway.d/*.conf; + # index.html must NEVER be cached: it's the SPA entrypoint that points at # the content-hashed JS/CSS chunks below. If the browser caches it, a new # deploy's hashed chunks are never discovered and the user keeps running diff --git a/frontend/src/lib/entrypoint.test.ts b/frontend/src/lib/entrypoint.test.ts index 7a889774c..fc2dac586 100644 --- a/frontend/src/lib/entrypoint.test.ts +++ b/frontend/src/lib/entrypoint.test.ts @@ -26,7 +26,8 @@ function run(env: Record) { ); const patched = script .replaceAll('/usr/share/nginx/html', html) - .replaceAll('/etc/nginx/conf.d/default.conf', conf); + .replaceAll('/etc/nginx/conf.d/default.conf', conf) + .replaceAll('/etc/nginx/gateway.d', path.join(dir, 'gateway.d')); const r = spawnSync('sh', ['-c', patched], { env: { PATH: process.env.PATH ?? '', ...env }, encoding: 'utf-8', diff --git a/frontend/src/lib/gatewayConf.test.ts b/frontend/src/lib/gatewayConf.test.ts new file mode 100644 index 000000000..ceb3de809 --- /dev/null +++ b/frontend/src/lib/gatewayConf.test.ts @@ -0,0 +1,216 @@ +/** + * Gateway mode (OP_GATEWAY_SUBPATHS): the entrypoint renders the sub-path + * locations for the four monitoring UIs into nginx's gateway.d, and removes + * them otherwise. Runs the real docker-entrypoint.sh against the real + * nginx.conf + nginx-gateway.conf in a temp tree, so no container is needed. + */ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const root = process.cwd(); +const script = readFileSync(path.resolve(root, 'docker-entrypoint.sh'), 'utf-8'); +const nginxConf = readFileSync(path.resolve(root, 'nginx.conf'), 'utf-8'); +const gatewayConf = readFileSync(path.resolve(root, 'nginx-gateway.conf'), 'utf-8'); + +function render(env: Record) { + const dir = mkdtempSync(path.join(tmpdir(), 'gateway-')); + const html = path.join(dir, 'html'); + const gatewayDir = path.join(dir, 'gateway.d'); + const snippets = path.join(dir, 'snippets'); + mkdirSync(html); + mkdirSync(gatewayDir); + mkdirSync(snippets); + const conf = path.join(dir, 'default.conf'); + writeFileSync(conf, nginxConf); + writeFileSync(path.join(snippets, 'gateway-subpaths.conf'), gatewayConf); + const patched = script + .replaceAll('/usr/share/nginx/html', html) + .replaceAll('/etc/nginx/conf.d/default.conf', conf) + .replaceAll('/etc/nginx/gateway.d', gatewayDir) + .replaceAll( + '/etc/nginx/snippets/gateway-subpaths.conf', + path.join(snippets, 'gateway-subpaths.conf'), + ); + const r = spawnSync('sh', ['-c', patched], { + env: { PATH: process.env.PATH ?? '', ...env }, + encoding: 'utf-8', + }); + const out = path.join(gatewayDir, 'gateway-subpaths.conf'); + return { + status: r.status, + stderr: r.stderr, + conf: readFileSync(conf, 'utf-8'), + gateway: existsSync(out) ? readFileSync(out, 'utf-8') : null, + }; +} + +function locations(text: string): Map { + const out = new Map(); + for (const m of text.matchAll(/^\s*location ([^{]+)\{([^}]*)\}/gm)) + out.set(m[1]!.trim(), m[2]!); + return out; +} + +const FORWARD = [ + 'proxy_set_header Host $http_host;', + 'proxy_set_header X-Forwarded-Host $http_host;', + 'proxy_set_header X-Forwarded-Proto $scheme;', + 'proxy_set_header X-Real-IP $remote_addr;', + 'proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;', +]; + +describe('gateway sub-path locations', () => { + it('nginx.conf includes the gateway directory', () => { + expect(nginxConf).toContain('include /etc/nginx/gateway.d/*.conf;'); + }); + + for (const off of [ + {}, + { OP_GATEWAY_SUBPATHS: 'false' }, + { OP_GATEWAY_SUBPATHS: '' }, + ] as Record[]) { + it(`renders nothing when disabled (${JSON.stringify(off)})`, () => { + const r = render(off); + expect(r.status).toBe(0); + expect(r.gateway).toBeNull(); + }); + } + + for (const on of ['true', 'TRUE', '1', 'yes', 'on']) { + it(`renders the locations when OP_GATEWAY_SUBPATHS=${on}`, () => { + const r = render({ OP_GATEWAY_SUBPATHS: on }); + expect(r.status).toBe(0); + expect(r.gateway).not.toBeNull(); + expect(r.gateway).not.toMatch(/__[A-Z_]+__/); + }); + } + + it('proxies each UI under its sub-path to the compose service by default', () => { + const g = render({ OP_GATEWAY_SUBPATHS: 'true' }).gateway!; + expect(g).toContain('set $grafana_upstream http://grafana:3000;'); + expect(g).toContain('set $prometheus_upstream http://prometheus:9090;'); + expect(g).toContain('set $dashboards_upstream http://opensearch-dashboards:5601;'); + expect(g).toContain('set $mlflow_upstream http://curation-mlflow:5000;'); + const locs = locations(g); + for (const [loc, upstream] of [ + ['^~ /grafana/', '$grafana_upstream'], + ['^~ /prometheus/', '$prometheus_upstream'], + ['^~ /dashboards/', '$dashboards_upstream'], + ['^~ /mlflow/', '$mlflow_upstream'], + ] as const) { + expect(locs.get(loc), loc).toContain(`proxy_pass ${upstream}`); + } + for (const bare of ['grafana', 'prometheus', 'dashboards', 'mlflow']) { + expect(locs.get(`= /${bare}`), bare).toContain(`return 301 /${bare}/;`); + } + }); + + it('forwards Host and X-Forwarded-* on every UI location', () => { + const g = render({ OP_GATEWAY_SUBPATHS: 'true' }).gateway!; + const proxied = [...locations(g)].filter(([, body]) => body.includes('proxy_pass')); + expect(proxied.length).toBeGreaterThanOrEqual(5); + for (const [loc, body] of proxied) { + for (const h of FORWARD) { + // MLflow's DNS-rebinding guard gets its own upstream Host (see below). + if (loc.includes('/mlflow/') && h === 'proxy_set_header Host $http_host;') + continue; + expect(body, `${loc} lacks ${h}`).toContain(h); + } + } + }); + + it('upgrades websockets for Grafana Live only, with HTTP/1.1', () => { + const g = render({ OP_GATEWAY_SUBPATHS: 'true' }).gateway!; + const locs = locations(g); + const live = locs.get('^~ /grafana/api/live/'); + expect(live).toBeDefined(); + expect(live).toContain('proxy_http_version 1.1;'); + expect(live).toContain('proxy_set_header Upgrade $http_upgrade;'); + expect(live).toContain('proxy_set_header Connection "upgrade";'); + expect(live).toContain('proxy_pass $grafana_upstream'); + expect(locs.get('^~ /grafana/')).not.toContain('Upgrade'); + }); + + it('strips the /mlflow prefix and sends the upstream Host so the allowed-hosts guard passes', () => { + const mlflow = locations(render({ OP_GATEWAY_SUBPATHS: 'true' }).gateway!).get( + '^~ /mlflow/', + )!; + expect(mlflow).toContain('rewrite ^/mlflow/(.*)$ /$1 break;'); + expect(mlflow).toContain('proxy_set_header Host $proxy_host;'); + expect(mlflow).toContain('proxy_set_header X-Forwarded-Host $http_host;'); + }); + + it('keeps the prefix for Grafana, Prometheus and Dashboards (they serve from it)', () => { + const locs = locations(render({ OP_GATEWAY_SUBPATHS: 'true' }).gateway!); + for (const loc of ['^~ /grafana/', '^~ /prometheus/', '^~ /dashboards/']) { + expect(locs.get(loc), loc).not.toContain('rewrite'); + } + }); + + it('honours per-service upstream overrides', () => { + const g = render({ + OP_GATEWAY_SUBPATHS: 'true', + GRAFANA_UPSTREAM: 'http://gf:3001', + MLFLOW_UPSTREAM: 'http://ml:5001', + }).gateway!; + expect(g).toContain('set $grafana_upstream http://gf:3001;'); + expect(g).toContain('set $mlflow_upstream http://ml:5001;'); + }); + + // Fail closed: a UI location may only reach a single-label compose service + // name over plain http, never an IP, an FQDN or another scheme. + for (const bad of [ + 'http://10.0.0.5:3000', + 'http://example.com:3000', + 'https://grafana:3000', + 'http://grafana', + 'http://grafana:3000/x', + 'http://127.0.0.1:3000', + 'http://grafana:3000;x', + 'http://[::1]:3000', + ]) { + it(`refuses GRAFANA_UPSTREAM=${bad} in gateway mode`, () => { + const r = render({ OP_GATEWAY_SUBPATHS: 'true', GRAFANA_UPSTREAM: bad }); + expect(r.status).not.toBe(0); + expect(r.stderr).toContain('GRAFANA_UPSTREAM'); + expect(r.gateway).toBeNull(); + }); + } + + it('every rendered upstream is a bare compose service name', () => { + const g = render({ OP_GATEWAY_SUBPATHS: 'true' }).gateway!; + const upstreams = [...g.matchAll(/set \$\w+_upstream (\S+);/g)].map((m) => m[1]!); + expect(upstreams).toHaveLength(4); + for (const u of upstreams) expect(u).toMatch(/^http:\/\/[a-z][a-z0-9-]*:\d{1,5}$/); + }); + + it('rejects an unparseable OP_GATEWAY_SUBPATHS rather than guessing', () => { + const r = render({ OP_GATEWAY_SUBPATHS: 'maybe' }); + expect(r.status).not.toBe(0); + expect(r.stderr).toContain('OP_GATEWAY_SUBPATHS'); + }); + + it('redirects are relative: nginx listens on 8080, the published port differs', () => { + expect(nginxConf).toContain('absolute_redirect off;'); + }); + + it('leaves the API docs locations at the root', () => { + const locs = locations(nginxConf); + for (const loc of [ + '= /docs', + '= /redoc', + '= /openapi.json', + '= /docs/oauth2-redirect', + '^~ /docs-assets/', + ]) { + expect(locs.get(loc), loc).toContain('proxy_pass $api_upstream'); + } + }); + + it('no hardcoded published host ports (4604-4609) in the gateway config', () => { + expect(gatewayConf).not.toMatch(/\b46\d\d\b/); + }); +}); From 81ec5d8c6df56093dc2845b8b3a8d3c1802a1d2e Mon Sep 17 00:00:00 2001 From: davidamacey Date: Fri, 9 Oct 2026 05:32:10 -0400 Subject: [PATCH 3/4] feat(compose): serve the UIs from sub-paths behind OP_GATEWAY_SUBPATHS --- CHANGELOG.md | 18 ++ INSTALLATION.md | 4 + SECURITY.md | 14 +- docker-compose.yml | 72 +++++++- docs-site/docs/operations/monitoring.mdx | 50 +++++- env.template | 19 +- tests/test_compose_gateway.py | 214 +++++++++++++++++++++++ 7 files changed, 386 insertions(+), 5 deletions(-) create mode 100644 tests/test_compose_gateway.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e2cf2f4..b9966f153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,24 @@ history of this codebase and was never published. This release is `[0.4.1]`. ## [Unreleased] +### Added + +- Gateway mode (#93): `OP_GATEWAY_SUBPATHS=true` makes Cropwright's nginx serve + Grafana, Prometheus, OpenSearch Dashboards and MLflow under `/grafana/`, + `/prometheus/`, `/dashboards/` and `/mlflow/` of its one published port, so the UIs + stay on `127.0.0.1`. One switch in `docker-compose.yml` starts each UI serving from + its sub-path, the API serves the path-relative `resource_links` and MLflow run + links (an explicit `OP_*_URL` still wins and a `0` port still hides the link), and + nginx upgrades Grafana Live websockets and forwards `Host` and `X-Forwarded-*`. + Fail closed: the UIs refuse to start when published beyond loopback, and Grafana + refuses the default or an empty admin password. Prometheus and MLflow still have no + authentication (`SECURITY.md`). The link audit is in the monitoring guide. + +### Fixed + +- Cropwright's nginx now answers redirects with relative `Location` headers; the + `/OpenProcessor` redirect used to name the container port 8080. + ### Changed - The experiment-tracking server and the trainer client move to MLflow 3.17 together diff --git a/INSTALLATION.md b/INSTALLATION.md index f3cb08a8f..882da1bb0 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -129,6 +129,10 @@ everything on 127.0.0.1.** `0.0.0.0`, so LAN browsers can follow resource links while the datastore and model APIs stay local. Change `GF_SECURITY_ADMIN_PASSWORD` first; MLflow and Prometheus have no auth. +- `OP_GATEWAY_SUBPATHS=true` (in `.env`, with the `cropwright` profile) serves the + four monitoring UIs under `/grafana/`, `/prometheus/`, `/dashboards/` and + `/mlflow/` on Cropwright's one port, so the UIs can stay on loopback; it needs a + non-default `GF_SECURITY_ADMIN_PASSWORD`. See the monitoring guide. - A re-run keeps the Cropwright bind you chose last time. Details: [SECURITY.md](SECURITY.md). diff --git a/SECURITY.md b/SECURITY.md index 9d1fc8566..d95fc81fb 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -71,7 +71,19 @@ the files it covers. Release signing is follow-up work. - **Grafana** ships with the default credentials `admin` / `admin` (`GF_SECURITY_ADMIN_USER` / `GF_SECURITY_ADMIN_PASSWORD` in `docker-compose.yml`). Change these before exposing Grafana to - anyone but yourself. + anyone but yourself. In gateway mode (`OP_GATEWAY_SUBPATHS=true`) Grafana + refuses to start with the default or an empty password. The variable only + sets the password when the `grafana_data` volume is first created; on an + existing volume change it with + `docker compose exec grafana grafana cli admin reset-admin-password `. +- **Prometheus and MLflow** have no authentication at all. In gateway mode + (see [the monitoring guide](docs-site/docs/operations/monitoring.mdx)) anyone + who can reach the gateway port can read Prometheus and read and write MLflow + experiments and registered models, exactly as they can already use the API. The + gateway moves these from "one port each" to "behind the one gateway port"; it + adds no login. Publish the gateway only on a trusted network or behind an + authenticating reverse proxy. Gateway mode also refuses to start the four UIs + when they are published beyond loopback. - **OpenSearch** ships with its security plugin disabled by default (`DISABLE_SECURITY_PLUGIN=true` / `DISABLE_SECURITY_DASHBOARDS_PLUGIN=true` in `docker-compose.yml`, commented "Disable security for dev (enable diff --git a/docker-compose.yml b/docker-compose.yml index 8687e04ae..8cd1b5647 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,49 @@ x-logging: &default-logging max-size: "10m" max-file: "5" +# Gateway mode (OP_GATEWAY_SUBPATHS=true): the Cropwright nginx serves Grafana, +# Prometheus, OpenSearch Dashboards and MLflow under /grafana/, /prometheus/, +# /dashboards/ and /mlflow/ of its one published port. Each UI must then serve +# FROM its sub-path, which Grafana and OpenSearch Dashboards take as env and +# Prometheus as flags (MLflow is left rooted and the gateway strips /mlflow: +# --static-prefix would also prefix /health and /api and break the in-network +# MLFLOW_TRACKING_URI). All four start through this one wrapper so the safety +# checks below apply to each: `sh -c