Apply native website noindex from Launchplane production state

docker/scripts/odoo_website_settings.py

@@ -4,7 +4,7 @@
 import json
 import re
 from typing import Any
-from urllib.parse import urlsplit
+from urllib.parse import SplitResult, urlsplit
 
 PRODUCTION_INSTANCE_NAMES = frozenset({"prod", "production"})
 INDEXING_PARAMETER = "cm_website.public_indexing_enabled"
@@ -20,6 +20,126 @@
     }
 )
 NATIVE_FIELDS = ("google_analytics_key", "cookies_bar", "block_third_party_domains")
+MANAGED_NOINDEX = '<meta name="robots" content="noindex" data-launchplane-production-use="prelaunch"/>'
+INDEXING_FIELDS = frozenset({"website_id", "public_origin", "production_use"})
+
+
+def indexing_head(head: str, production_use: str) -> str:
+    unmanaged = head.removeprefix(MANAGED_NOINDEX)
+    if "data-launchplane-production-use" in unmanaged.lower():
+        raise RuntimeError("Native website head has an ambiguous managed indexing element.")
+    return MANAGED_NOINDEX + unmanaged if production_use == "prelaunch" else unmanaged
+
+
+def head_digest(head: str) -> str:
+    return hashlib.sha256(head.encode()).hexdigest()
+
+
+def parse_public_origin(origin: object) -> SplitResult:
+    if not isinstance(origin, str):
+        raise RuntimeError("Website settings require a public HTTPS origin.")
+    parsed = urlsplit(origin)
+    if (
+        parsed.scheme != "https"
+        or not parsed.hostname
+        or parsed.username
+        or parsed.password
+        or parsed.path not in {"", "/"}
+        or parsed.query
+        or parsed.fragment
+    ):
+        raise RuntimeError("Website settings require a public HTTPS origin.")
+    return parsed
+
+
+def indexing_website(env: Any, raw: dict[str, Any]) -> Any:
+    website_id = raw["website_id"]
+    if type(website_id) is not int or website_id < 1:
+        raise RuntimeError("Website indexing requires one native website ID.")
+    origin = raw["public_origin"]
+    if not isinstance(origin, str) or any(character.isspace() for character in origin):
+        raise RuntimeError("Website indexing requires an exact public HTTPS origin.")
+    try:
+        parsed = parse_public_origin(origin)
+        valid_port = parsed.port is None or 0 < parsed.port <= 65535
+    except ValueError as error:
+        raise RuntimeError("Website indexing requires an exact public HTTPS origin.") from error
+    if parsed.username is not None or not valid_port:
+        raise RuntimeError("Website indexing requires an exact public HTTPS origin.")
+    websites = env["website"].sudo()
+    host = str(parsed.netloc).lower().encode("idna")
+    matches = [
+        website for website in websites.search([]) if str(urlsplit(str(website.domain or "")).netloc).lower().encode("idna") == host
+    ]
+    if len(matches) != 1 or matches[0].id != website_id:
+        raise RuntimeError("Website indexing requires an unambiguous native website/origin binding.")
+    website = matches[0]
+    if str(website.domain or "").rstrip("/") != origin.rstrip("/"):
+        raise RuntimeError("Website indexing origin does not match the native website domain.")
+    if "custom_code_head" not in website._fields:
+        raise RuntimeError("Native website head setting is unavailable.")
+    return website
+
+
+def plan_website_indexing(env: Any, payload: dict[str, object] | None, *, instance: str) -> dict[str, Any] | None:
+    raw = payload.get("website_indexing") if payload else None
+    if raw is None:
+        return None
+    if (
+        not payload
+        or instance.strip().lower() not in PRODUCTION_INSTANCE_NAMES
+        or payload.get("instance") not in PRODUCTION_INSTANCE_NAMES
+    ):
+        raise RuntimeError("Website indexing requires a matching production payload.")
+    if not isinstance(raw, dict) or set(raw) != INDEXING_FIELDS or raw["production_use"] not in ("prelaunch", "live"):
+        raise RuntimeError("Invalid Launchplane website indexing actuation data.")
+    legacy = payload.get("website_settings")
+    if legacy is not None and (
+        not isinstance(legacy, dict)
+        or legacy.get("website_id") != raw["website_id"]
+        or str(legacy.get("public_origin", "")).rstrip("/") != str(raw["public_origin"]).rstrip("/")
+    ):
+        raise RuntimeError("Website settings and indexing must select the same native website and origin.")
+    if not website_is_loaded(env):
+        raise RuntimeError("Website indexing requires the installed website addon.")
+    website = indexing_website(env, raw)
+    before = str(website.custom_code_head or "")
+    after = indexing_head(before, raw["production_use"])
+    return {
+        **raw,
+        "field": "custom_code_head",
+        "before_sha256": head_digest(before),
+        "after_sha256": head_digest(after),
+        "changed": before != after,
+        "managed_noindex_before": before.startswith(MANAGED_NOINDEX),
+        "managed_noindex_after": after.startswith(MANAGED_NOINDEX),
+    }
+
+
+def apply_website_settings(env: Any, payload: dict[str, object] | None, *, instance: str) -> str | None:
+    plan = plan_website_indexing(env, payload, instance=instance)
+    if plan is None:
+        return apply_legacy_website_settings(env, payload, instance=instance)
+    website = indexing_website(env, plan)
+    try:
+        with env.cr.savepoint():
+            digest = apply_legacy_website_settings(env, payload, instance=instance)
+            before = str(website.custom_code_head or "")
+            if head_digest(before) != plan["before_sha256"]:
+                raise RuntimeError("Native website head changed after indexing planning.")
+            after = indexing_head(before, plan["production_use"])
+            if before != after:
+                website.write({"custom_code_head": after or False})
+            website.flush_recordset(["custom_code_head"])
+            website.invalidate_recordset(["custom_code_head"])
+            if head_digest(str(website.custom_code_head or "")) != plan["after_sha256"]:
+                raise RuntimeError("Native website indexing readback failed.")
+    except BaseException:
+        if clear_cache := getattr(env.registry, "clear_cache", None):
+            clear_cache("stable")
+        raise
+    print("website_indexing=" + json.dumps({**plan, "readback_verified": True}, sort_keys=True, separators=(",", ":")))
+    return digest
 
 
 def read_string_parameter(parameters: Any, key: str) -> str:
@@ -60,7 +180,7 @@ def exclude_collection(websites: Any, parameters: Any) -> None:
         raise RuntimeError("Nonproduction website indexing exclusion readback failed.")
 
 
-def apply_website_settings(env: Any, payload: dict[str, object] | None, *, instance: str) -> str | None:
+def apply_legacy_website_settings(env: Any, payload: dict[str, object] | None, *, instance: str) -> str | None:
     """Call in the existing environment-overrides transaction; caller commits.
 
     Empty/unknown instances are nonproduction. Clearing every copied website is
@@ -107,19 +227,7 @@ def apply_website_settings(env: Any, payload: dict[str, object] | None, *, insta
     if analytics and not (raw["cookies_bar"] and raw["block_third_party_domains"]):
         raise RuntimeError("Website analytics requires the cookie bar and third-party blocking.")
     origin = raw["public_origin"]
-    if not isinstance(origin, str):
-        raise RuntimeError("Website settings require a public HTTPS origin.")
-    parsed = urlsplit(origin)
-    if (
-        parsed.scheme != "https"
-        or not parsed.hostname
-        or parsed.username
-        or parsed.password
-        or parsed.path not in {"", "/"}
-        or parsed.query
-        or parsed.fragment
-    ):
-        raise RuntimeError("Website settings require a public HTTPS origin.")
+    parse_public_origin(origin)
     if operation == "disable_collection":
         # Historical recovery has no verified selector. Disable collection across
         # the database while preserving every website's consent/embedding choices.

docs/tooling/workspace-cli.md

@@ -455,6 +455,58 @@ Notes
   lacks Website logs the skip and continues. These flags are
   runtime assertions supplied by Launchplane-managed records or operator input;
   local runtimes remain optional unless a caller explicitly sets them.
+- Launchplane may include `website_indexing` alongside the unchanged
+  `website_settings` in the existing post-deploy payload:
+
+  ```json
+  {
+    "instance": "prod",
+    "website_indexing": {
+      "website_id": 1,
+      "public_origin": "https://site.example.test",
+      "production_use": "prelaunch"
+    }
+  }
+  ```
+
+  These three fields are required and no extra fields are accepted. They are
+  actuation data derived from Launchplane's current production state and exact
+  production website binding, never an operator SEO flag. Only `prelaunch` and
+  `live` are accepted, on a production instance with a matching production
+  envelope. Omission preserves legacy behavior. The selected native website
+  must have that exact HTTPS domain (an optional trailing slash is ignored),
+  with no second website competing for the host and port even under another
+  scheme. When both payload objects are present, their website ID and origin
+  must agree before either can write.
+  Missing addons, fields, selectors and ambiguous managed markup refuse.
+
+  `plan_website_indexing(env, payload, instance=...)` is the read-only dry-run
+  function for Launchplane's generator to invoke; this does not add a CLI or
+  enable a target operation. The producer must accept the new payload object
+  and regenerate its embedded consumer from the reviewed devkit commit.
+  It returns the selector, origin, supplied state, native field, SHA-256 of the
+  current and proposed head, `changed`, and managed-element presence before and
+  after. `apply_website_settings` uses that same plan, preserves the caller's
+  transaction, verifies native database readback and prints `website_indexing=`
+  followed by that JSON plus `readback_verified`. It never commits. This is
+  tentative transaction evidence until the caller commits successfully; the
+  legacy return digest and `website_settings_sha256` retain their meaning.
+  A savepoint rolls back this consumer's writes on failure, and an outer
+  transaction rollback also restores the prior head. A rollback after an
+  already committed transition must supply the authoritative restored state
+  through this same consumer; no recorded production state is inferred here.
+
+  Prelaunch prepends one identifiable robots `noindex` meta element to native
+  `website.custom_code_head`; live removes only that exact managed prefix.
+  All pre-existing head content is retained. Indexing-only payloads do not
+  change analytics, consent, robots.txt, domain, routes, pages or other sites.
+  Live removes only this protection; independent existing indexing choices,
+  including the legacy accepted setting, continue to apply.
+  The setting covers HTML pages rendering the native website layout, not every
+  downloadable resource. It neither removes sitemap entries nor proves removal
+  from search results. Existing robots rules remain unchanged: crawlers must
+  be able to fetch a page to observe its noindex directive, as described by
+  [Google](https://developers.google.com/search/docs/crawling-indexing/block-indexing).
 - Legacy setting-shaped inputs such as `ENV_OVERRIDE_CONFIG_PARAM__*`,
   `ENV_OVERRIDE_AUTHENTIK__*`, and `ENV_OVERRIDE_SHOPIFY__*` are still accepted
   as a compatibility input and converted into the same typed payload, but they

tests/live_odoo/addons/website_settings_checks/tests/__init__.py

@@ -1 +1 @@
-from . import test_website_settings
+from . import test_website_indexing, test_website_settings

tests/live_odoo/addons/website_settings_checks/tests/test_website_indexing.py

@@ -0,0 +1,143 @@
+import hashlib
+import io
+import json
+from contextlib import redirect_stdout
+from typing import Any
+from unittest.mock import patch
+
+from lxml import html
+from odoo import tools
+from odoo.tests import HttpCase, tagged
+from odoo_website_settings import (
+    INDEXING_PARAMETER,
+    apply_website_settings,
+    plan_website_indexing,
+    read_string_parameter,
+    write_string_parameter,
+)
+
+
+@tagged("post_install", "-at_install")
+class TestNativeWebsiteIndexing(HttpCase):
+    def test_public_noindex_preserves_routes_robots_and_unmanaged_head(self) -> None:
+        self.enterContext(patch.dict(tools.config.options, {"proxy_mode": True}))
+        original = '<meta name="robots" content="nofollow"><script>window.synthetic = "preserved";</script>'
+        website = self.env["website"].create(
+            {
+                "name": "Synthetic indexing",
+                "domain": "https://indexing.example.test",
+                "custom_code_head": original,
+                "robots_txt": "User-agent: *\nDisallow: /private-fixture\n",
+                "google_analytics_key": "G-SYNTHETIC",
+                "cookies_bar": True,
+                "block_third_party_domains": True,
+            }
+        )
+        other = self.env["website"].create(
+            {
+                "name": "Unrelated synthetic",
+                "domain": "https://unrelated.example.test",
+                "custom_code_head": original,
+            }
+        )
+        payload: dict[str, Any] = {
+            "instance": "prod",
+            "website_indexing": {
+                "website_id": website.id,
+                "public_origin": website.domain,
+                "production_use": "prelaunch",
+            },
+        }
+
+        def page(host: str = "indexing.example.test", path: str = "/contactus") -> str:
+            response = self.url_open(path, headers={"X-Forwarded-Host": host, "X-Forwarded-Proto": "https"})
+            self.assertEqual(response.status_code, 200)
+            return response.text
+
+        def managed_count(document: str) -> int:
+            return sum(
+                "noindex" in content.lower().replace(",", " ").split()
+                for content in html.fromstring(document).xpath('//head/meta[@name="robots"]/@content')
+            )
+
+        def apply() -> dict[str, object]:
+            output = io.StringIO()
+            with redirect_stdout(output):
+                apply_website_settings(self.env, payload, instance="prod")
+            return json.loads(output.getvalue().split("website_indexing=")[1].splitlines()[0])
+
+        baseline = page()
+        self.assertEqual(managed_count(baseline), 0)
+        canonical = html.fromstring(baseline).xpath('//link[@rel="canonical"]/@href')
+        self.assertTrue(canonical)
+        robots = page(path="/robots.txt")
+        self.assertNotIn("Disallow: /\n", robots)
+        before = website.read(
+            ["domain", "custom_code_head", "google_analytics_key", "cookies_bar", "block_third_party_domains", "robots_txt"]
+        )
+        plan = plan_website_indexing(self.env, payload, instance="prod")
+        self.assertEqual(website.read(list(before[0])), before)
+        receipt = apply()
+        self.assertEqual(receipt["after_sha256"], plan["after_sha256"])
+        self.env.cr.execute("SELECT custom_code_head FROM website WHERE id=%s", (website.id,))
+        protected = self.env.cr.fetchone()[0]
+        self.assertEqual(hashlib.sha256(protected.encode()).hexdigest(), receipt["after_sha256"])
+        protected_page = page()
+        self.assertEqual(managed_count(protected_page), 1)
+        self.assertEqual(html.fromstring(protected_page).xpath('//head/meta[@content="nofollow"]/@name'), ["robots"])
+        self.assertIn('window.synthetic = "preserved";', protected_page)
+        self.assertEqual(html.fromstring(protected_page).xpath('//link[@rel="canonical"]/@href'), canonical)
+        self.assertEqual(page(path="/robots.txt"), robots)
+        self.assertEqual(managed_count(page("unrelated.example.test")), 0)
+        self.assertEqual(other.custom_code_head, original)
+        self.assertFalse(apply()["changed"])
+        payload["website_indexing"]["production_use"] = "live"
+        parameters = self.env["ir.config_parameter"].sudo()
+        write_string_parameter(parameters, INDEXING_PARAMETER, "0")
+        payload["website_settings"] = {
+            "operation": "apply",
+            "website_id": website.id,
+            "public_origin": website.domain,
+            "public_indexing_enabled": True,
+            "google_analytics_key": "G-CHANGED",
+            "cookies_bar": True,
+            "block_third_party_domains": True,
+        }
+        native_write = type(website).write
+
+        def ignore_head(records: Any, values: dict[str, object]) -> bool:
+            if "custom_code_head" in values:
+                return True
+            return native_write(records, values)
+
+        with patch.object(type(website), "write", ignore_head), self.assertRaises(RuntimeError):
+            apply()
+        self.assertEqual(website.custom_code_head, protected)
+        self.assertEqual(website.google_analytics_key, before[0]["google_analytics_key"])
+        self.assertEqual(read_string_parameter(parameters, INDEXING_PARAMETER), "0")
+        del payload["website_settings"]
+        self.assertEqual(managed_count(page()), 1)
+        try:
+            with self.env.cr.savepoint():
+                apply()
+                raise RuntimeError("synthetic release failure")
+        except RuntimeError as error:
+            self.assertEqual(str(error), "synthetic release failure")
+        self.assertEqual(website.custom_code_head, protected)
+        self.assertEqual(managed_count(page()), 1)
+        apply()
+        self.assertEqual(website.read(list(before[0])), before)
+        self.assertEqual(managed_count(page()), 0)
+        self.assertEqual(page(path="/robots.txt"), robots)
+        payload["website_indexing"]["production_use"] = "prelaunch"
+        apply()
+        self.assertEqual(managed_count(page()), 1)
+        duplicate = self.env["website"].create({"name": "Ambiguous synthetic", "domain": "http://indexing.example.test"})
+        with self.assertRaises(RuntimeError):
+            apply()
+        duplicate.unlink()
+        payload["website_indexing"]["public_origin"] = other.domain
+        with self.assertRaises(RuntimeError):
+            apply()
+        self.assertEqual(website.custom_code_head, protected)
+        print("NATIVE_INDEXING_HTTP_PROOF " + json.dumps(receipt, sort_keys=True), flush=True)