diff --git a/deploy/apache/beta-bot-blocks.conf b/deploy/apache/beta-bot-blocks.conf index 86c47568..c2c5e347 100644 --- a/deploy/apache/beta-bot-blocks.conf +++ b/deploy/apache/beta-bot-blocks.conf @@ -13,10 +13,20 @@ # correct than a list of names that grows every month. # # NOTE ON IP RULES: this host sits behind Cloudflare, so every request arrives -# from a Cloudflare address. `Require not ip` and %{REMOTE_ADDR} therefore match -# Cloudflare, not the client -- they block nothing or everything. Only header -# matching (User-Agent below) is reliable until mod_remoteip is configured with -# Cloudflare's ranges; see the bottom of this file. +# from a Cloudflare address. Left alone, `Require not ip` and %{REMOTE_ADDR} +# would match Cloudflare rather than the client, and block nothing or everything. +# +# mod_remoteip IS configured on the dev box -- `RemoteIPHeader CF-Connecting-IP` +# with the trusted ranges, and the `cloudflare` LogFormat recording the restored +# address. Verified 2026-09-16. This paragraph previously said it was not, which +# is worth a correction rather than a quiet edit: it is the reason the nginx +# translation in deploy/nginx first shipped without real-IP handling, and the +# rate limit that depends on it would have covered the whole internet with one +# bucket. +# +# The rules below still match on User-Agent rather than address. That is a +# deliberate choice, not a limitation: a crawler's address changes and its +# self-description mostly does not. RewriteEngine On @@ -73,6 +83,9 @@ # # a2enmod remoteip && systemctl reload apache2 # +# ALREADY APPLIED on the dev box; kept here as the record of what was done and +# what to reapply if this host is rebuilt. +# # # RemoteIPHeader CF-Connecting-IP # RemoteIPTrustedProxy 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md new file mode 100644 index 00000000..2aefe38c --- /dev/null +++ b/deploy/nginx/README.md @@ -0,0 +1,165 @@ +# nginx for reactome.org + +Four environments, one set of shared routes. **None of this is in use yet** — +beta is still served by hand-configured Apache on the dev box. This exists so +the configuration is reviewable, diffable and rebuildable, and so the site can +eventually be started locally. + +``` +common/ what every environment shares +local.conf your machine compose service names, no TLS +dev.conf beta.reactome.org the development box +release.conf release.reactome.org the staged release +production.conf reactome.org the public site +``` + +## What differs, and why + +| | local | dev | release | production | +| --------------------------------------- | ------------- | ------- | ------- | ---------- | +| TLS | no | yes | yes | yes | +| Real client IPs via Cloudflare | no | yes | yes | yes | +| Named AI/SEO crawlers blocked | no | yes | yes | **yes** | +| _Anything_ calling itself a bot blocked | no | yes | yes | **NO** | +| Tina admin | **reachable** | denied | denied | denied | +| Rate limit | none | 100 r/s | 100 r/s | 600 r/s | +| Unknown hostnames | — | 503 | 503 | 503 | + +Two rows carry the whole risk of this arrangement. + +**The blanket bot rule must never reach production.** It blocks Googlebot and +Bingbot along with everything else. Safe on a host that must not be indexed, +catastrophic on the one that must. It lives in `common/block-all-automation.conf` +and is included by `dev` and `release` only. Verified by test, not by reading: + +``` +dev Googlebot → 403 GPTBot → 403 +production Googlebot → allowed GPTBot → 403 +``` + +**The admin rule is repeated in each environment rather than shared.** nginx +cannot choose an include by variable, and the workaround would have hidden the +one decision that must be obvious at a glance: whether a CMS editor is reachable +from the internet. It is reachable locally — editing content is the point of +running locally — and denied everywhere else. + +## Verified + +Against a running `nginx:alpine`, per environment: + +``` +/admin → 403 /adminfoo → passes through (matches Apache) +/admin/cms → 403 /administer → passes through (matches Apache) +GPTBot → 403 CUBOT phone → passes through +curl → passes through dev.reactome.org → 503 +www.reactome.org → 301 to reactome.org +``` + +`/adminfoo` is in that list because the first draft blocked it: `location ^~ +/admin` is a prefix match, while Apache's `LocationMatch "^/admin(/|$)"` is not. +Found by asking the running config rather than by reading it. + +## Local is not runnable yet, and says so + +`local.conf` expects compose services named `app`, `content-service`, +`deltasignal` and `chatbot`. **Only `app` exists** in `docker-compose.yml` today. +So this file is the shape of the answer, not the answer: starting the site +locally still needs those services defined. + +DeltaSignal and the chatbot are resolved _per request_ rather than at startup, +through a variable and a resolver, so their absence gives a 502 on those two +routes instead of stopping nginx entirely. Named in an upstream block they would +be resolved at startup, and one missing service would refuse to start the whole +site — which is a poor welcome for someone who only wanted to look at a pathway. + +`content-service` is deliberately **not** treated that way: a site with no content +service is not worth starting, and failing loudly is the right answer. + +## Upstreams move; that is expected + +`common/routes.conf` names `site`, `content`, `analysis`, `deltasignal` and +`chatbot`. Each environment defines them. Two changes are coming and this shape +absorbs both: node in this repository taking over most of what Tomcat serves, and +the site being startable locally, where they become compose service names. + +`/api/` for DeltaSignal exists in none of the Apache configuration, which is why +DeltaSignal cannot work on beta today — the Angular side calls it as a bare +relative path, routed only by the dev server's proxy, which does not exist in a +built artifact. Note the port on the dev box: **8090**, not the 8080 DeltaSignal's +own compose binds, because 8080 there is Tomcat. + +## Connections, which is a reason this is worth doing + +Measured on the dev box, sockets to Tomcat's 8080: + +``` +76 CLOSE-WAIT 19 ESTABLISHED +``` + +Four in five connections leaked. Whether Apache or Tomcat is at fault is not +settled and does not need to be — the pairing produces it, and it goes when +Tomcat does. + +What this configuration changes is that connection handling is chosen rather than +inherited: every upstream sets `keepalive`, with `keepalive_timeout 10s` — shorter +than Tomcat's 20s default, so the backend never closes a pooled socket first. + +That pairing is not optional. `proxy_http_version 1.1` with `Connection ""` and +**no** `keepalive` tells the backend to hold the socket open while nginx has no +pool to keep it in, which is a way of causing this pile rather than curing it. An +earlier draft of these files did exactly that — described the pooling and +configured none of it — and it was caught by reviewing the configuration against +its own comments. + +## Certificates: only the servers have them + +`local.conf` is plain HTTP on port 80 and touches no certificate at all — +verified, it starts with nothing mounted at `/etc/letsencrypt`. Running the site +on your own machine should not require obtaining a certificate for a hostname you +do not own. + +Deployed environments present certificates the server already holds. This box +needs exactly two: + +| | | +| ------------------- | ----------------------------------------------- | +| `beta.reactome.org` | the site | +| `dev.reactome.org` | the retired host, so its 503 is not a TLS error | + +and eventually only one, when the Angular site is the one going forward. The +wikis live on the release machine; `login.dev` is gone. The `reactome.org` +certificate here is presented only because the retired vhost lists that name as +an alias — production serves it. + +## Before any of this serves traffic + +**Certbot first, and proved.** Renewal is already automated — `certbot.timer` +twice daily and a `/etc/cron.d/certbot` besides, both running `certbot -q renew`. +That is the risk rather than the reassurance: `renew` uses each certificate's +_stored_ authenticator, and `beta.reactome.org` still stores +`authenticator = apache`. The day Apache stops, that renewal begins failing, and +`-q` means it fails without saying anything. The certificate expires 60-odd days +later. + +Only beta needs moving: `dev.reactome.org` already renews via `dns-cloudflare`, +which is the same path, already working on this machine. + +The safe path is already proven on that box: `dev.reactome.org` renews via +`dns-cloudflare`, and `python3-certbot-dns-cloudflare` is installed. +`python3-certbot-nginx` is **not** — which is why `common/tls.conf` writes the TLS +settings out rather than including `/etc/letsencrypt/options-ssl-nginx.conf`, a +file that does not exist here. A missing include stops nginx from starting, and a +cutover is the worst moment to learn that. + +``` +certbot renew --cert-name beta.reactome.org --dry-run # must pass first +``` + +Then: ports 80 and 443 handed over atomically, with a one-command rollback, and +**not during a curator review cycle** — beta is the QA gate. + +## What is deliberately absent + +No `docker-compose` service. Adding one would invite `docker compose up` to take +port 443 from Apache on a box where that is the live site. It belongs in the same +change as the cutover. diff --git a/deploy/nginx/common/block-ai-crawlers.conf b/deploy/nginx/common/block-ai-crawlers.conf new file mode 100644 index 00000000..7e53b6a9 --- /dev/null +++ b/deploy/nginx/common/block-ai-crawlers.conf @@ -0,0 +1,14 @@ +# Named AI and SEO crawlers, blocked on every host. +# +# Safe for production: this list contains no search engine anyone wants to be +# indexed by. It is the same list the Apache configuration carries, kept in step +# deliberately -- add here and there together. +# +# What is NOT here, and must not be added here, is the rule that blocks anything +# self-identifying as a bot. That one is safe on beta precisely because beta must +# not be indexed, and would deindex reactome.org. It lives in the beta config +# alone, with the reason beside it. +map $http_user_agent $blocked_crawler { + default 0; + "~*(Amzn-SearchBot|Amazonbot|PetalBot|AhrefsBot|SemrushBot|DataForSeo|MJ12bot|DotBot|CCBot|Barkrowler|Bytespider|GPTBot|ClaudeBot|Claude-Web|anthropic-ai|PerplexityBot|Bytedance|SeekportBot|serpstatbot|ZoominfoBot|Diffbot|ImagesiftBot|Timpibot|Omgilibot|meta-externalagent|Applebot-Extended)" 1; +} diff --git a/deploy/nginx/common/block-all-automation.conf b/deploy/nginx/common/block-all-automation.conf new file mode 100644 index 00000000..87579024 --- /dev/null +++ b/deploy/nginx/common/block-all-automation.conf @@ -0,0 +1,24 @@ +# Block anything self-identifying as automation. +# +# ONLY for hosts that must not be indexed — the dev box and release. It blocks +# Googlebot and Bingbot along with everything else, so including it in +# production would deindex reactome.org. That is not a hypothetical caution: the +# Apache configuration this came from carries the same warning in the same words. +# +# Safe on a non-indexed host precisely because the default there is "no +# automation at all", which is a far easier rule to keep correct than a list of +# names that grows every month. +# +# CUBOT is an Android phone brand whose user agent contains "bot", so it is +# excluded first: a rule that blocks a real device is worse than one that misses +# a crawler. Word boundaries are not an option — nearly every crawler is +# "bot", so \bbot\b would match almost none of them. +# +# curl and wget are deliberately absent: they are how these hosts are checked +# from outside, and a blocked health check is worse than a scraped page. +map $http_user_agent $blocked_automation { + default 0; + "~*(python-requests|python-urllib|aiohttp|httpx|Go-http-client|Java/|okhttp|libwww-perl|Scrapy|node-fetch|axios)" 1; + "~*CUBOT" 0; + "~*(bot|crawler|crawling|spider|scraper|fetcher|archiver|indexer)" 1; +} diff --git a/deploy/nginx/common/cloudflare-real-ip.conf b/deploy/nginx/common/cloudflare-real-ip.conf new file mode 100644 index 00000000..fba5223e --- /dev/null +++ b/deploy/nginx/common/cloudflare-real-ip.conf @@ -0,0 +1,36 @@ +# Real client IPs behind Cloudflare. +# +# Every request arrives from a Cloudflare address, so without this $remote_addr +# is Cloudflare's: logs describe Cloudflare, and any rate limit keyed on the +# client address covers the entire internet with one bucket. +# +# These ranges are copied from the Apache `mod_remoteip` configuration already +# running on the dev box, which uses the same header and the same list. They are +# not invented, and they are not permanent -- Cloudflare publishes changes at +# https://www.cloudflare.com/ips/ and a stale range silently stops resolving the +# clients behind it. +real_ip_header CF-Connecting-IP; +real_ip_recursive on; + +set_real_ip_from 173.245.48.0/20; +set_real_ip_from 103.21.244.0/22; +set_real_ip_from 103.22.200.0/22; +set_real_ip_from 103.31.4.0/22; +set_real_ip_from 141.101.64.0/18; +set_real_ip_from 108.162.192.0/18; +set_real_ip_from 190.93.240.0/20; +set_real_ip_from 188.114.96.0/20; +set_real_ip_from 197.234.240.0/22; +set_real_ip_from 198.41.128.0/17; +set_real_ip_from 162.158.0.0/15; +set_real_ip_from 104.16.0.0/13; +set_real_ip_from 104.24.0.0/14; +set_real_ip_from 172.64.0.0/13; +set_real_ip_from 131.0.72.0/22; +set_real_ip_from 2400:cb00::/32; +set_real_ip_from 2606:4700::/32; +set_real_ip_from 2803:f800::/32; +set_real_ip_from 2405:b500::/32; +set_real_ip_from 2405:8100::/32; +set_real_ip_from 2a06:98c0::/29; +set_real_ip_from 2c0f:f248::/32; diff --git a/deploy/nginx/common/routes-core.conf b/deploy/nginx/common/routes-core.conf new file mode 100644 index 00000000..8f9482c1 --- /dev/null +++ b/deploy/nginx/common/routes-core.conf @@ -0,0 +1,48 @@ +# The routes every environment shares EXCEPT the two a local machine may not be +# running: DeltaSignal and the chatbot. Deployed environments include +# routes.conf, which is this plus those two; local.conf includes this and +# defines them itself, resolved per request so a missing service does not stop +# nginx from starting. +# +# Upstreams are named, not addressed: each environment file defines `site`, +# `content`, `analysis`, `deltasignal` and `chatbot` to suit itself — localhost +# ports on a host, service names in compose, and something else again when node +# in this repository takes over what Tomcat serves. +# +# What is NOT here is anything that differs by environment: bot blocking, rate +# limiting, TLS, and the Tina admin all live in the environment files, because +# getting one of them wrong in the wrong place is the expensive kind of mistake. +# +# The admin rule in particular is deliberately repeated in each environment +# rather than shared. nginx cannot choose an include by variable, and the +# workaround would have hidden the one decision that must be obvious at a glance: +# whether a CMS editor is reachable from the internet. + +location /ContentService/ { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://content/ContentService/; + # A diagram or SBML export takes minutes. Apache's default is 300s and the + # site relies on it; nginx defaults to 60 and would cut them off. + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} + +location /AnalysisService/ { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://analysis/AnalysisService/; + # An analysis over a large identifier list is not quick either. + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} + + + +# The site itself. +location / { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://site/; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} diff --git a/deploy/nginx/common/routes.conf b/deploy/nginx/common/routes.conf new file mode 100644 index 00000000..de53f490 --- /dev/null +++ b/deploy/nginx/common/routes.conf @@ -0,0 +1,67 @@ +# The routes every environment shares. Included inside a server block. +# +# Upstreams are named, not addressed: each environment file defines `site`, +# `content`, `analysis`, `deltasignal` and `chatbot` to suit itself — localhost +# ports on a host, service names in compose, and something else again when node +# in this repository takes over what Tomcat serves. +# +# What is NOT here is anything that differs by environment: bot blocking, rate +# limiting, TLS, and the Tina admin all live in the environment files, because +# getting one of them wrong in the wrong place is the expensive kind of mistake. +# +# The admin rule in particular is deliberately repeated in each environment +# rather than shared. nginx cannot choose an include by variable, and the +# workaround would have hidden the one decision that must be obvious at a glance: +# whether a CMS editor is reachable from the internet. + +location /ContentService/ { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://content/ContentService/; + # A diagram or SBML export takes minutes. Apache's default is 300s and the + # site relies on it; nginx defaults to 60 and would cut them off. + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} + +location /AnalysisService/ { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://analysis/AnalysisService/; + # An analysis over a large identifier list is not quick either. + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} + +# DeltaSignal. Apache has no such route, which is why DeltaSignal cannot work on +# beta today: the Angular side calls /api/pathways, /api/parse and /api/solve as +# bare relative paths, routed only by proxy.conf.js — the dev server's proxy, +# which does not exist in a built artifact. +location /api/ { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://deltasignal/api/; + # Solving a perturbation is not a fast request. + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} + +# The chatbot. A plain proxy renders its UI and then hangs with no replies: it is +# server-sent events and needs buffering off as well as the upgrade headers. The +# Apache configuration records having hit exactly this. +location = /chat { return 302 /chat/; } +location /chat/ { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://chatbot/chat/; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_buffering off; + proxy_read_timeout 3600s; +} + +# The site itself. +location / { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://site/; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} diff --git a/deploy/nginx/common/tls.conf b/deploy/nginx/common/tls.conf new file mode 100644 index 00000000..de9dcf44 --- /dev/null +++ b/deploy/nginx/common/tls.conf @@ -0,0 +1,15 @@ +# TLS settings. +# +# Written out rather than `include /etc/letsencrypt/options-ssl-nginx.conf`, +# which is what certbot's nginx plugin would provide -- and that plugin is not +# installed on the dev box: only python3-certbot-apache and +# python3-certbot-dns-cloudflare are. A config that includes a file which does +# not exist does not start, and finding that out during a cutover is the worst +# time to find it out. +ssl_session_cache shared:le_nginx_SSL:10m; +ssl_session_timeout 1440m; +ssl_session_tickets off; + +ssl_protocols TLSv1.2 TLSv1.3; +ssl_prefer_server_ciphers off; +ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305"; diff --git a/deploy/nginx/common/upstream-proxy.conf b/deploy/nginx/common/upstream-proxy.conf new file mode 100644 index 00000000..f1b297f2 --- /dev/null +++ b/deploy/nginx/common/upstream-proxy.conf @@ -0,0 +1,44 @@ +# Shared proxy behaviour. Included inside a location. +# +# Connection handling is the part to read, because it is why this exists. +# +# Measured on the dev box 2026-09-16, sockets to Tomcat's 8080: +# +# 76 CLOSE-WAIT 19 ESTABLISHED +# +# CLOSE-WAIT means the far end sent FIN and the near end never closed. Four out +# of five connections to Tomcat were leaked file descriptors. Whether Apache or +# Tomcat is at fault is not settled and does not need to be: the pairing is what +# produces it, and this is the layer where the behaviour is chosen rather than +# inherited. +# +# nginx's default is HTTP/1.0 upstream with `Connection: close` -- a new +# connection per request, which cannot leak but pays setup on every one. +# +# The two directives below opt out of that, and they are only safe alongside a +# `keepalive` directive in the upstream block. Without one, clearing the +# Connection header tells the backend to hold the socket open while nginx has no +# pool to keep it in -- which is a way of *causing* the CLOSE-WAIT pile above, +# not curing it. An earlier draft of this file did exactly that: it described +# pooling and configured none. Every upstream now sets `keepalive`, and its +# timeout is shorter than the backend's so the backend never closes first. +proxy_http_version 1.1; +proxy_set_header Connection ""; + +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; + +# Timeouts are NOT set here. nginx rejects a duplicate directive in the same +# context, so a shared default plus a per-route override does not compile -- and +# the routes genuinely differ: a diagram export takes minutes, the chat holds a +# connection for an hour. Each location sets its own, which also means the value +# is visible where it matters rather than inherited from a file you have to go +# and read. + +# Do not hold a failed backend against the reader: retry the next upstream +# rather than returning its error, but never replay a request that is not safe +# to repeat. +proxy_next_upstream error timeout http_502 http_503 http_504; +proxy_next_upstream_tries 2; diff --git a/deploy/nginx/common/websocket.conf b/deploy/nginx/common/websocket.conf new file mode 100644 index 00000000..773a487b --- /dev/null +++ b/deploy/nginx/common/websocket.conf @@ -0,0 +1,8 @@ +# WebSocket upgrade, at http level. +# +# The Angular dev server and the chat both need it, and a proxy that does not +# pass these two headers renders the UI and then hangs with no replies. +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} diff --git a/deploy/nginx/dev.conf b/deploy/nginx/dev.conf new file mode 100644 index 00000000..6dde7fa4 --- /dev/null +++ b/deploy/nginx/dev.conf @@ -0,0 +1,156 @@ +# beta.reactome.org, on the development box. +# +# The strictest of the four: this host must not be indexed at all, and each +# pathway URL walked is a Neo4j query against a Tomcat heap that fills within +# minutes. So the default is "no automation", not "no bad automation". + +upstream site { + server 127.0.0.1:4200; # the built app, via serve-prod.js + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream content { + server 127.0.0.1:8080; # Tomcat today, node later + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream analysis { + server 127.0.0.1:8080; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream deltasignal { + server 127.0.0.1:8090; # NOT 8080: that is Tomcat here + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream chatbot { + server 127.0.0.1:8000; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} + +include /etc/nginx/common/websocket.conf; +include /etc/nginx/common/cloudflare-real-ip.conf; +include /etc/nginx/common/block-ai-crawlers.conf; +include /etc/nginx/common/block-all-automation.conf; + +# Replaces mod_evasive, which is active on this host: 12 requests per page per +# second, 100 per site, blocked for 60. Keyed on the client address, which is +# only meaningful because the real-IP block above is included — without it every +# request appears to come from Cloudflare and one bucket covers the internet. +limit_req_zone $binary_remote_addr zone=dev_rate:10m rate=100r/s; +limit_conn_zone $binary_remote_addr zone=dev_conn:10m; + +# dev.reactome.org stays retired, and so does anything else pointed here. +# +# This box needs exactly two certificates -- dev.reactome.org and +# beta.reactome.org -- and eventually only one, when the Angular site is the one +# going forward. The wikis are on the release machine and login.dev is gone. +# +# That vhost is a Joomla 3.10.12 stack, end of life since 2023, which was the +# entry vector on production and release in September 2026. Under Apache the only +# thing making this host safe is that it returns 503 for everything. If nginx +# answered only for beta, a request for dev.reactome.org — or reactome.org, which +# is an alias of that same vhost — would fall to the default server. +# +# It is also the best argument for this migration: that vhost still carries +# `Options +Includes` and `XBitHack on` with mod_include loaded. nginx has no +# mod_include, so the Server-Side Includes hole cannot exist here at all. +# dev.reactome.org, retired, with its own certificate. +# +# It has one, and it already renews via dns-cloudflare -- the only certificate on +# this box that does. Presenting beta's certificate for this name would be a TLS +# mismatch, which is a worse answer than the clean 503 the retired host is meant +# to give. +server { + listen 80; + listen 443 ssl; + server_name dev.reactome.org; + ssl_certificate /etc/letsencrypt/live/dev.reactome.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/dev.reactome.org/privkey.pem; + include /etc/nginx/common/tls.conf; + return 503; +} + +# Anything else pointed at this box. +# +# A name we do not serve cannot be answered without a certificate mismatch +# whatever we present, so beta's is used and the answer is 503 regardless. The +# point is that it is never silently served beta's content. +server { + listen 80 default_server; + listen 443 ssl default_server; + server_name _; + ssl_certificate /etc/letsencrypt/live/beta.reactome.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/beta.reactome.org/privkey.pem; + include /etc/nginx/common/tls.conf; + return 503; +} + +server { + listen 80; + server_name beta.reactome.org; + location /.well-known/acme-challenge/ { root /var/www/certbot; } + location / { return 301 https://$host$request_uri; } +} + +server { + listen 443 ssl; + http2 on; + server_name beta.reactome.org; + + ssl_certificate /etc/letsencrypt/live/beta.reactome.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/beta.reactome.org/privkey.pem; + include /etc/nginx/common/tls.conf; + + # The Tina admin, denied: this host faces the internet. + # + # A regex, not `^~ /admin`, which is a prefix match and would also catch + # `/adminfoo` — a URL the Apache this replaces serves. Its rule is + # `LocationMatch "^/admin(/|$)"`, so this matches the same two shapes. + # A regex location beats the `/` catch-all regardless of order. + location ~ ^/admin(/|$) { return 403; } + + if ($blocked_crawler) { return 403; } + if ($blocked_automation) { return 403; } + + # burst absorbs the dozen parallel requests one diagram legitimately makes; + # nodelay so a real reader is not queued behind their own page. + limit_req zone=dev_rate burst=100 nodelay; + limit_conn dev_conn 24; + + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml + application/xml application/xml+rss text/javascript image/svg+xml; + gzip_min_length 1024; + + client_max_body_size 64m; + + include /etc/nginx/common/routes.conf; +} diff --git a/deploy/nginx/local.conf b/deploy/nginx/local.conf new file mode 100644 index 00000000..24b5751c --- /dev/null +++ b/deploy/nginx/local.conf @@ -0,0 +1,91 @@ +# The website on your own machine. +# +# No TLS, no Cloudflare, no bot blocking, no rate limiting, and the Tina admin is +# reachable — because none of those protect anything here and every one of them +# is an obstacle to the thing this exists for: someone being able to start the +# site and look at it. +# +# Upstreams are compose service names. Change them here and nowhere else. + +upstream site { + server app:4200; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream content { + server content-service:8080; # Tomcat today, node later + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream analysis { + server content-service:8080; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} + +include /etc/nginx/common/websocket.conf; + +# Docker's internal DNS. Needed because the two optional services below are +# resolved per request rather than at startup. +resolver 127.0.0.11 valid=30s ipv6=off; + +server { + listen 80; + server_name localhost; + + # The Tina admin is reachable here, and deliberately so: editing content is + # the point of running locally. Every deployed environment denies it. + location ~ ^/admin(/|$) { + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://site; + } + + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml + application/xml application/xml+rss text/javascript image/svg+xml; + gzip_min_length 1024; + + # DeltaSignal and the chatbot are optional here: someone starting the site + # to look at a pathway should not need a Julia solver and a chat service + # running as well. + # + # Resolved through a variable, per request, so nginx starts whether or not + # they exist. Named directly in an upstream block they would be resolved at + # startup, and a missing one stops nginx entirely -- the whole site refusing + # to start because a service nobody asked for is absent. + location /api/ { + include /etc/nginx/common/upstream-proxy.conf; + set $deltasignal_upstream deltasignal:8080; + proxy_pass http://$deltasignal_upstream/api/; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + } + + location = /chat { return 302 /chat/; } + location /chat/ { + include /etc/nginx/common/upstream-proxy.conf; + set $chatbot_upstream chatbot:8000; + proxy_pass http://$chatbot_upstream/chat/; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_buffering off; + proxy_read_timeout 3600s; + } + + include /etc/nginx/common/routes-core.conf; +} diff --git a/deploy/nginx/production.conf b/deploy/nginx/production.conf new file mode 100644 index 00000000..4a23e5d1 --- /dev/null +++ b/deploy/nginx/production.conf @@ -0,0 +1,131 @@ +# reactome.org — the public site. +# +# THE IMPORTANT DIFFERENCE: this file does NOT include block-all-automation.conf. +# +# That rule blocks anything self-identifying as a bot, which is safe on a host +# that must not be indexed and catastrophic here: it blocks Googlebot and +# Bingbot along with the rest, and Reactome needs to be found. The Apache +# configuration it came from carries the same warning. Only the named AI and SEO +# crawlers are refused. +# +# Traffic here is the reason nginx is worth the change at all. On the dev box the +# choice is marginal; at production volume it is not, and the connection handling +# in common/upstream-proxy.conf is deliberate rather than inherited — see the +# CLOSE-WAIT measurement recorded there. + +upstream site { + server 127.0.0.1:4200; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream content { + server 127.0.0.1:8080; # Tomcat today, node later + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream analysis { + server 127.0.0.1:8080; # the analysis maths stays Java + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream deltasignal { + server 127.0.0.1:8090; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream chatbot { + server 127.0.0.1:8000; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} + +include /etc/nginx/common/websocket.conf; +include /etc/nginx/common/cloudflare-real-ip.conf; +include /etc/nginx/common/block-ai-crawlers.conf; + +# Higher than the non-public hosts, because real readers arrive in numbers here +# and a limit tuned for a single-user dev box would throttle a lecture theatre. +# Still present: a crawler that ignores robots.txt costs the same Neo4j queries +# whichever host it walks. +limit_req_zone $binary_remote_addr zone=prod_rate:20m rate=600r/s; +limit_conn_zone $binary_remote_addr zone=prod_conn:20m; + +server { + listen 80 default_server; + listen 443 ssl default_server; + server_name _; + ssl_certificate /etc/letsencrypt/live/reactome.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/reactome.org/privkey.pem; + include /etc/nginx/common/tls.conf; + return 503; +} + +server { + listen 80; + server_name reactome.org www.reactome.org; + location /.well-known/acme-challenge/ { root /var/www/certbot; } + location / { return 301 https://reactome.org$request_uri; } +} + +server { + listen 443 ssl; + http2 on; + server_name www.reactome.org; + ssl_certificate /etc/letsencrypt/live/reactome.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/reactome.org/privkey.pem; + include /etc/nginx/common/tls.conf; + # One canonical host, so links and search results do not split between two. + return 301 https://reactome.org$request_uri; +} + +server { + listen 443 ssl; + http2 on; + server_name reactome.org; + + ssl_certificate /etc/letsencrypt/live/reactome.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/reactome.org/privkey.pem; + include /etc/nginx/common/tls.conf; + + # Denied: this host faces the internet. + location ~ ^/admin(/|$) { return 403; } + + if ($blocked_crawler) { return 403; } + + limit_req zone=prod_rate burst=300 nodelay; + limit_conn prod_conn 64; + + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml + application/xml application/xml+rss text/javascript image/svg+xml; + gzip_min_length 1024; + + client_max_body_size 64m; + + include /etc/nginx/common/routes.conf; +} diff --git a/deploy/nginx/release.conf b/deploy/nginx/release.conf new file mode 100644 index 00000000..fc0a1b12 --- /dev/null +++ b/deploy/nginx/release.conf @@ -0,0 +1,112 @@ +# release.reactome.org — the staged release, before it is promoted. +# +# Same posture as the dev box, for the same reason: this host must not be +# indexed. A staged release appearing in search results is worse than beta doing +# so, because its content looks authoritative and its URLs stop working the day +# it is promoted. +# +# It differs from dev in what it points at, not in how it behaves. + +upstream site { + server 127.0.0.1:4200; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream content { + server 127.0.0.1:8080; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream analysis { + server 127.0.0.1:8080; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream deltasignal { + server 127.0.0.1:8090; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} +upstream chatbot { + server 127.0.0.1:8000; + # Pooled, and the timeout is the load-bearing number: it MUST be shorter + # than the backend's own idle timeout, or the backend closes first and this + # pool hands out sockets that are already gone. Tomcat's keepAliveTimeout + # defaults to 20s, so 10s here leaves room. + keepalive 32; + keepalive_timeout 10s; + keepalive_requests 1000; +} + +include /etc/nginx/common/websocket.conf; +include /etc/nginx/common/cloudflare-real-ip.conf; +include /etc/nginx/common/block-ai-crawlers.conf; +include /etc/nginx/common/block-all-automation.conf; + +limit_req_zone $binary_remote_addr zone=rel_rate:10m rate=100r/s; +limit_conn_zone $binary_remote_addr zone=rel_conn:10m; + +server { + listen 80 default_server; + listen 443 ssl default_server; + server_name _; + ssl_certificate /etc/letsencrypt/live/release.reactome.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/release.reactome.org/privkey.pem; + include /etc/nginx/common/tls.conf; + return 503; +} + +server { + listen 80; + server_name release.reactome.org; + location /.well-known/acme-challenge/ { root /var/www/certbot; } + location / { return 301 https://$host$request_uri; } +} + +server { + listen 443 ssl; + http2 on; + server_name release.reactome.org; + + ssl_certificate /etc/letsencrypt/live/release.reactome.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/release.reactome.org/privkey.pem; + include /etc/nginx/common/tls.conf; + + # Denied: this host faces the internet. + location ~ ^/admin(/|$) { return 403; } + + if ($blocked_crawler) { return 403; } + if ($blocked_automation) { return 403; } + + limit_req zone=rel_rate burst=100 nodelay; + limit_conn rel_conn 24; + + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml + application/xml application/xml+rss text/javascript image/svg+xml; + gzip_min_length 1024; + + client_max_body_size 64m; + + include /etc/nginx/common/routes.conf; +}