diff --git a/docker/scripts/odoo_website_settings.py b/docker/scripts/odoo_website_settings.py index 937233d..fcb77ec 100644 --- a/docker/scripts/odoo_website_settings.py +++ b/docker/scripts/odoo_website_settings.py @@ -4,7 +4,7 @@ import json import re from typing import Any -from urllib.parse import urlsplit +from urllib.parse import SplitResult, urlsplit PRODUCTION_INSTANCE_NAMES = frozenset({"prod", "production"}) INDEXING_PARAMETER = "cm_website.public_indexing_enabled" @@ -20,6 +20,126 @@ } ) NATIVE_FIELDS = ("google_analytics_key", "cookies_bar", "block_third_party_domains") +MANAGED_NOINDEX = '' +INDEXING_FIELDS = frozenset({"website_id", "public_origin", "production_use"}) + + +def indexing_head(head: str, production_use: str) -> str: + unmanaged = head.removeprefix(MANAGED_NOINDEX) + if "data-launchplane-production-use" in unmanaged.lower(): + raise RuntimeError("Native website head has an ambiguous managed indexing element.") + return MANAGED_NOINDEX + unmanaged if production_use == "prelaunch" else unmanaged + + +def head_digest(head: str) -> str: + return hashlib.sha256(head.encode()).hexdigest() + + +def parse_public_origin(origin: object) -> SplitResult: + if not isinstance(origin, str): + raise RuntimeError("Website settings require a public HTTPS origin.") + parsed = urlsplit(origin) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.path not in {"", "/"} + or parsed.query + or parsed.fragment + ): + raise RuntimeError("Website settings require a public HTTPS origin.") + return parsed + + +def indexing_website(env: Any, raw: dict[str, Any]) -> Any: + website_id = raw["website_id"] + if type(website_id) is not int or website_id < 1: + raise RuntimeError("Website indexing requires one native website ID.") + origin = raw["public_origin"] + if not isinstance(origin, str) or any(character.isspace() for character in origin): + raise RuntimeError("Website indexing requires an exact public HTTPS origin.") + try: + parsed = parse_public_origin(origin) + valid_port = parsed.port is None or 0 < parsed.port <= 65535 + except ValueError as error: + raise RuntimeError("Website indexing requires an exact public HTTPS origin.") from error + if parsed.username is not None or not valid_port: + raise RuntimeError("Website indexing requires an exact public HTTPS origin.") + websites = env["website"].sudo() + host = str(parsed.netloc).lower().encode("idna") + matches = [ + website for website in websites.search([]) if str(urlsplit(str(website.domain or "")).netloc).lower().encode("idna") == host + ] + if len(matches) != 1 or matches[0].id != website_id: + raise RuntimeError("Website indexing requires an unambiguous native website/origin binding.") + website = matches[0] + if str(website.domain or "").rstrip("/") != origin.rstrip("/"): + raise RuntimeError("Website indexing origin does not match the native website domain.") + if "custom_code_head" not in website._fields: + raise RuntimeError("Native website head setting is unavailable.") + return website + + +def plan_website_indexing(env: Any, payload: dict[str, object] | None, *, instance: str) -> dict[str, Any] | None: + raw = payload.get("website_indexing") if payload else None + if raw is None: + return None + if ( + not payload + or instance.strip().lower() not in PRODUCTION_INSTANCE_NAMES + or payload.get("instance") not in PRODUCTION_INSTANCE_NAMES + ): + raise RuntimeError("Website indexing requires a matching production payload.") + if not isinstance(raw, dict) or set(raw) != INDEXING_FIELDS or raw["production_use"] not in ("prelaunch", "live"): + raise RuntimeError("Invalid Launchplane website indexing actuation data.") + legacy = payload.get("website_settings") + if legacy is not None and ( + not isinstance(legacy, dict) + or legacy.get("website_id") != raw["website_id"] + or str(legacy.get("public_origin", "")).rstrip("/") != str(raw["public_origin"]).rstrip("/") + ): + raise RuntimeError("Website settings and indexing must select the same native website and origin.") + if not website_is_loaded(env): + raise RuntimeError("Website indexing requires the installed website addon.") + website = indexing_website(env, raw) + before = str(website.custom_code_head or "") + after = indexing_head(before, raw["production_use"]) + return { + **raw, + "field": "custom_code_head", + "before_sha256": head_digest(before), + "after_sha256": head_digest(after), + "changed": before != after, + "managed_noindex_before": before.startswith(MANAGED_NOINDEX), + "managed_noindex_after": after.startswith(MANAGED_NOINDEX), + } + + +def apply_website_settings(env: Any, payload: dict[str, object] | None, *, instance: str) -> str | None: + plan = plan_website_indexing(env, payload, instance=instance) + if plan is None: + return apply_legacy_website_settings(env, payload, instance=instance) + website = indexing_website(env, plan) + try: + with env.cr.savepoint(): + digest = apply_legacy_website_settings(env, payload, instance=instance) + before = str(website.custom_code_head or "") + if head_digest(before) != plan["before_sha256"]: + raise RuntimeError("Native website head changed after indexing planning.") + after = indexing_head(before, plan["production_use"]) + if before != after: + website.write({"custom_code_head": after or False}) + website.flush_recordset(["custom_code_head"]) + website.invalidate_recordset(["custom_code_head"]) + if head_digest(str(website.custom_code_head or "")) != plan["after_sha256"]: + raise RuntimeError("Native website indexing readback failed.") + except BaseException: + if clear_cache := getattr(env.registry, "clear_cache", None): + clear_cache("stable") + raise + print("website_indexing=" + json.dumps({**plan, "readback_verified": True}, sort_keys=True, separators=(",", ":"))) + return digest def read_string_parameter(parameters: Any, key: str) -> str: @@ -60,7 +180,7 @@ def exclude_collection(websites: Any, parameters: Any) -> None: raise RuntimeError("Nonproduction website indexing exclusion readback failed.") -def apply_website_settings(env: Any, payload: dict[str, object] | None, *, instance: str) -> str | None: +def apply_legacy_website_settings(env: Any, payload: dict[str, object] | None, *, instance: str) -> str | None: """Call in the existing environment-overrides transaction; caller commits. Empty/unknown instances are nonproduction. Clearing every copied website is @@ -107,19 +227,7 @@ def apply_website_settings(env: Any, payload: dict[str, object] | None, *, insta if analytics and not (raw["cookies_bar"] and raw["block_third_party_domains"]): raise RuntimeError("Website analytics requires the cookie bar and third-party blocking.") origin = raw["public_origin"] - if not isinstance(origin, str): - raise RuntimeError("Website settings require a public HTTPS origin.") - parsed = urlsplit(origin) - if ( - parsed.scheme != "https" - or not parsed.hostname - or parsed.username - or parsed.password - or parsed.path not in {"", "/"} - or parsed.query - or parsed.fragment - ): - raise RuntimeError("Website settings require a public HTTPS origin.") + parse_public_origin(origin) if operation == "disable_collection": # Historical recovery has no verified selector. Disable collection across # the database while preserving every website's consent/embedding choices. diff --git a/docs/tooling/workspace-cli.md b/docs/tooling/workspace-cli.md index 0efad34..1161d70 100644 --- a/docs/tooling/workspace-cli.md +++ b/docs/tooling/workspace-cli.md @@ -455,6 +455,58 @@ Notes lacks Website logs the skip and continues. These flags are runtime assertions supplied by Launchplane-managed records or operator input; local runtimes remain optional unless a caller explicitly sets them. +- Launchplane may include `website_indexing` alongside the unchanged + `website_settings` in the existing post-deploy payload: + + ```json + { + "instance": "prod", + "website_indexing": { + "website_id": 1, + "public_origin": "https://site.example.test", + "production_use": "prelaunch" + } + } + ``` + + These three fields are required and no extra fields are accepted. They are + actuation data derived from Launchplane's current production state and exact + production website binding, never an operator SEO flag. Only `prelaunch` and + `live` are accepted, on a production instance with a matching production + envelope. Omission preserves legacy behavior. The selected native website + must have that exact HTTPS domain (an optional trailing slash is ignored), + with no second website competing for the host and port even under another + scheme. When both payload objects are present, their website ID and origin + must agree before either can write. + Missing addons, fields, selectors and ambiguous managed markup refuse. + + `plan_website_indexing(env, payload, instance=...)` is the read-only dry-run + function for Launchplane's generator to invoke; this does not add a CLI or + enable a target operation. The producer must accept the new payload object + and regenerate its embedded consumer from the reviewed devkit commit. + It returns the selector, origin, supplied state, native field, SHA-256 of the + current and proposed head, `changed`, and managed-element presence before and + after. `apply_website_settings` uses that same plan, preserves the caller's + transaction, verifies native database readback and prints `website_indexing=` + followed by that JSON plus `readback_verified`. It never commits. This is + tentative transaction evidence until the caller commits successfully; the + legacy return digest and `website_settings_sha256` retain their meaning. + A savepoint rolls back this consumer's writes on failure, and an outer + transaction rollback also restores the prior head. A rollback after an + already committed transition must supply the authoritative restored state + through this same consumer; no recorded production state is inferred here. + + Prelaunch prepends one identifiable robots `noindex` meta element to native + `website.custom_code_head`; live removes only that exact managed prefix. + All pre-existing head content is retained. Indexing-only payloads do not + change analytics, consent, robots.txt, domain, routes, pages or other sites. + Live removes only this protection; independent existing indexing choices, + including the legacy accepted setting, continue to apply. + The setting covers HTML pages rendering the native website layout, not every + downloadable resource. It neither removes sitemap entries nor proves removal + from search results. Existing robots rules remain unchanged: crawlers must + be able to fetch a page to observe its noindex directive, as described by + [Google](https://developers.google.com/search/docs/crawling-indexing/block-indexing). - Legacy setting-shaped inputs such as `ENV_OVERRIDE_CONFIG_PARAM__*`, `ENV_OVERRIDE_AUTHENTIK__*`, and `ENV_OVERRIDE_SHOPIFY__*` are still accepted as a compatibility input and converted into the same typed payload, but they diff --git a/tests/live_odoo/addons/website_settings_checks/tests/__init__.py b/tests/live_odoo/addons/website_settings_checks/tests/__init__.py index 5c23aea..acc39c4 100644 --- a/tests/live_odoo/addons/website_settings_checks/tests/__init__.py +++ b/tests/live_odoo/addons/website_settings_checks/tests/__init__.py @@ -1 +1 @@ -from . import test_website_settings +from . import test_website_indexing, test_website_settings diff --git a/tests/live_odoo/addons/website_settings_checks/tests/test_website_indexing.py b/tests/live_odoo/addons/website_settings_checks/tests/test_website_indexing.py new file mode 100644 index 0000000..8debde2 --- /dev/null +++ b/tests/live_odoo/addons/website_settings_checks/tests/test_website_indexing.py @@ -0,0 +1,143 @@ +import hashlib +import io +import json +from contextlib import redirect_stdout +from typing import Any +from unittest.mock import patch + +from lxml import html +from odoo import tools +from odoo.tests import HttpCase, tagged +from odoo_website_settings import ( + INDEXING_PARAMETER, + apply_website_settings, + plan_website_indexing, + read_string_parameter, + write_string_parameter, +) + + +@tagged("post_install", "-at_install") +class TestNativeWebsiteIndexing(HttpCase): + def test_public_noindex_preserves_routes_robots_and_unmanaged_head(self) -> None: + self.enterContext(patch.dict(tools.config.options, {"proxy_mode": True})) + original = '' + website = self.env["website"].create( + { + "name": "Synthetic indexing", + "domain": "https://indexing.example.test", + "custom_code_head": original, + "robots_txt": "User-agent: *\nDisallow: /private-fixture\n", + "google_analytics_key": "G-SYNTHETIC", + "cookies_bar": True, + "block_third_party_domains": True, + } + ) + other = self.env["website"].create( + { + "name": "Unrelated synthetic", + "domain": "https://unrelated.example.test", + "custom_code_head": original, + } + ) + payload: dict[str, Any] = { + "instance": "prod", + "website_indexing": { + "website_id": website.id, + "public_origin": website.domain, + "production_use": "prelaunch", + }, + } + + def page(host: str = "indexing.example.test", path: str = "/contactus") -> str: + response = self.url_open(path, headers={"X-Forwarded-Host": host, "X-Forwarded-Proto": "https"}) + self.assertEqual(response.status_code, 200) + return response.text + + def managed_count(document: str) -> int: + return sum( + "noindex" in content.lower().replace(",", " ").split() + for content in html.fromstring(document).xpath('//head/meta[@name="robots"]/@content') + ) + + def apply() -> dict[str, object]: + output = io.StringIO() + with redirect_stdout(output): + apply_website_settings(self.env, payload, instance="prod") + return json.loads(output.getvalue().split("website_indexing=")[1].splitlines()[0]) + + baseline = page() + self.assertEqual(managed_count(baseline), 0) + canonical = html.fromstring(baseline).xpath('//link[@rel="canonical"]/@href') + self.assertTrue(canonical) + robots = page(path="/robots.txt") + self.assertNotIn("Disallow: /\n", robots) + before = website.read( + ["domain", "custom_code_head", "google_analytics_key", "cookies_bar", "block_third_party_domains", "robots_txt"] + ) + plan = plan_website_indexing(self.env, payload, instance="prod") + self.assertEqual(website.read(list(before[0])), before) + receipt = apply() + self.assertEqual(receipt["after_sha256"], plan["after_sha256"]) + self.env.cr.execute("SELECT custom_code_head FROM website WHERE id=%s", (website.id,)) + protected = self.env.cr.fetchone()[0] + self.assertEqual(hashlib.sha256(protected.encode()).hexdigest(), receipt["after_sha256"]) + protected_page = page() + self.assertEqual(managed_count(protected_page), 1) + self.assertEqual(html.fromstring(protected_page).xpath('//head/meta[@content="nofollow"]/@name'), ["robots"]) + self.assertIn('window.synthetic = "preserved";', protected_page) + self.assertEqual(html.fromstring(protected_page).xpath('//link[@rel="canonical"]/@href'), canonical) + self.assertEqual(page(path="/robots.txt"), robots) + self.assertEqual(managed_count(page("unrelated.example.test")), 0) + self.assertEqual(other.custom_code_head, original) + self.assertFalse(apply()["changed"]) + payload["website_indexing"]["production_use"] = "live" + parameters = self.env["ir.config_parameter"].sudo() + write_string_parameter(parameters, INDEXING_PARAMETER, "0") + payload["website_settings"] = { + "operation": "apply", + "website_id": website.id, + "public_origin": website.domain, + "public_indexing_enabled": True, + "google_analytics_key": "G-CHANGED", + "cookies_bar": True, + "block_third_party_domains": True, + } + native_write = type(website).write + + def ignore_head(records: Any, values: dict[str, object]) -> bool: + if "custom_code_head" in values: + return True + return native_write(records, values) + + with patch.object(type(website), "write", ignore_head), self.assertRaises(RuntimeError): + apply() + self.assertEqual(website.custom_code_head, protected) + self.assertEqual(website.google_analytics_key, before[0]["google_analytics_key"]) + self.assertEqual(read_string_parameter(parameters, INDEXING_PARAMETER), "0") + del payload["website_settings"] + self.assertEqual(managed_count(page()), 1) + try: + with self.env.cr.savepoint(): + apply() + raise RuntimeError("synthetic release failure") + except RuntimeError as error: + self.assertEqual(str(error), "synthetic release failure") + self.assertEqual(website.custom_code_head, protected) + self.assertEqual(managed_count(page()), 1) + apply() + self.assertEqual(website.read(list(before[0])), before) + self.assertEqual(managed_count(page()), 0) + self.assertEqual(page(path="/robots.txt"), robots) + payload["website_indexing"]["production_use"] = "prelaunch" + apply() + self.assertEqual(managed_count(page()), 1) + duplicate = self.env["website"].create({"name": "Ambiguous synthetic", "domain": "http://indexing.example.test"}) + with self.assertRaises(RuntimeError): + apply() + duplicate.unlink() + payload["website_indexing"]["public_origin"] = other.domain + with self.assertRaises(RuntimeError): + apply() + self.assertEqual(website.custom_code_head, protected) + print("NATIVE_INDEXING_HTTP_PROOF " + json.dumps(receipt, sort_keys=True), flush=True) diff --git a/tests/test_website_indexing.py b/tests/test_website_indexing.py new file mode 100644 index 0000000..b409e19 --- /dev/null +++ b/tests/test_website_indexing.py @@ -0,0 +1,119 @@ +import copy +import hashlib +import io +import json +import unittest +from contextlib import contextmanager, redirect_stdout +from typing import Any + +from tests.test_odoo_website_settings import Environment, consumer, settings + + +class Cursor: + def __init__(self, env: Environment) -> None: + self.env = env + + @contextmanager + def savepoint(self) -> Any: + before = copy.deepcopy(self.env.websites.records) + parameters = copy.deepcopy(self.env.parameters.values) + try: + yield + except BaseException: + self.env.websites.records = before + self.env.parameters.values = parameters + raise + + +class IndexingEnvironment(Environment): + def __init__(self) -> None: + super().__init__() + self.cr = Cursor(self) + + +class WebsiteIndexingTests(unittest.TestCase): + def setUp(self) -> None: + self.env = IndexingEnvironment() + self.original = '' + for website in self.env.websites.records: + website._fields.add("custom_code_head") + website.custom_code_head = self.original + self.env.websites.records[1].domain = "https://unrelated.example" + self.payload: dict[str, Any] = { + "instance": "prod", + "website_indexing": {"website_id": 1, "public_origin": "https://site.example", "production_use": "prelaunch"}, + } + + def apply(self) -> dict[str, Any]: + output = io.StringIO() + with redirect_stdout(output): + consumer.apply_website_settings(self.env, self.payload, instance="prod") + return json.loads(output.getvalue().split("website_indexing=")[1].splitlines()[0]) + + def test_plan_is_read_only_and_apply_preserves_every_unmanaged_setting(self) -> None: + before = copy.deepcopy([vars(website) for website in self.env.websites.records]) + plan = consumer.plan_website_indexing(self.env, self.payload, instance="prod") + self.assertEqual(before, [vars(website) for website in self.env.websites.records]) + self.assertEqual(plan["before_sha256"], hashlib.sha256(self.original.encode()).hexdigest()) + receipt = self.apply() + website = self.env.websites.records[0] + self.assertEqual(receipt["after_sha256"], hashlib.sha256(website.custom_code_head.encode()).hexdigest()) + self.assertEqual(receipt["after_sha256"], plan["after_sha256"]) + self.assertTrue(receipt["managed_noindex_after"]) + self.assertTrue(website.custom_code_head.endswith(self.original)) + self.assertEqual(website.google_analytics_key, "G-COPIED") + self.assertEqual(website.domain, "https://site.example") + self.assertEqual(vars(self.env.websites.records[1]), before[1]) + self.assertEqual(self.env.parameters.values, {consumer.INDEXING_PARAMETER: "1"}) + self.assertFalse(self.apply()["changed"]) + self.payload["website_indexing"]["production_use"] = "live" + self.apply() + self.assertEqual(website.custom_code_head, self.original) + + def test_invalid_or_ambiguous_bindings_refuse_before_legacy_writes(self) -> None: + self.payload["website_settings"] = settings() + for change in ( + {"website_id": True}, + {"website_id": 99}, + {"public_origin": "https://wrong.example"}, + {"public_origin": "https://site.example/path"}, + {"production_use": "unknown"}, + {"extra": True}, + ): + with self.subTest(change=change): + payload = copy.deepcopy(self.payload) + payload["website_indexing"].update(change) + with self.assertRaises(RuntimeError): + consumer.apply_website_settings(self.env, payload, instance="prod") + self.assertFalse(any(website.writes for website in self.env.websites.records)) + self.env.websites.records[1].domain = "https://site.example/" + with self.assertRaises(RuntimeError): + consumer.apply_website_settings(self.env, self.payload, instance="prod") + self.assertFalse(any(website.writes for website in self.env.websites.records)) + + def test_failed_native_write_rolls_back_legacy_settings_and_keeps_protection(self) -> None: + self.apply() + protected = self.env.websites.records[0].custom_code_head + self.payload["website_indexing"]["production_use"] = "live" + self.payload["website_settings"] = settings() + self.env.parameters.values[consumer.INDEXING_PARAMETER] = "0" + self.env.websites.records[0].ignored_write_fields = {"custom_code_head"} + with self.assertRaises(RuntimeError): + self.apply() + self.assertEqual(self.env.websites.records[0].custom_code_head, protected) + self.assertEqual(self.env.websites.records[0].google_analytics_key, "G-COPIED") + self.assertEqual(self.env.parameters.values[consumer.INDEXING_PARAMETER], "0") + + def test_combined_payload_cannot_select_different_websites(self) -> None: + self.payload["website_settings"] = {**settings(), "website_id": 2, "public_origin": "https://unrelated.example"} + with self.assertRaises(RuntimeError): + consumer.apply_website_settings(self.env, self.payload, instance="prod") + self.assertFalse(any(website.writes for website in self.env.websites.records)) + + def test_nonproduction_payload_cannot_remove_protection(self) -> None: + self.apply() + protected = self.env.websites.records[0].custom_code_head + self.payload["website_indexing"]["production_use"] = "live" + with self.assertRaises(RuntimeError): + consumer.apply_website_settings(self.env, self.payload, instance="testing") + self.assertEqual(self.env.websites.records[0].custom_code_head, protected)