Bound native20 product-image conversion without losing source images

docker/scripts/nativeupgrade/website_sale/20.0.1.0/pre-product-images.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import runpy
+from pathlib import Path
+from typing import Any
+
+
+def migrate(cr: Any, version: str | None) -> None:
+    if not version or version.split(".", 1)[0] != "19":
+        return
+    root = Path(__file__).resolve().parents[3]
+    diagnostics = runpy.run_path(str(root / "migration_diagnostics.py"))["MigrationDiagnostics"]()
+    install = runpy.run_path(str(root / "product_images_20.py"))["install_bounded_image_conversion"]
+    diagnostics.check("Website Sale bounded image loader", install, None)

docker/scripts/product_images_20.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from functools import wraps
+from typing import Any
+
+IMAGE_BATCH_SIZE = 32
+
+
+@contextmanager
+def scope_native_template_search(owner: Any, selected: list[int]) -> Iterator[None]:
+    from odoo.fields import Domain
+
+    original_search = owner.search
+    inherited = "search" not in owner.__dict__
+    first_search = True
+
+    @wraps(original_search)
+    def search(records: Any, domain: Any, *args: Any, **kwargs: Any) -> Any:
+        nonlocal first_search
+        if first_search:
+            assert any(term.field_expr == "image_1920" for term in Domain(domain).iter_conditions())
+            first_search = False
+            domain = Domain(domain) & Domain("id", "in", selected)
+        return original_search(records, domain, *args, **kwargs)
+
+    owner.search = search
+    try:
+        yield
+    finally:
+        if inherited:
+            del owner.search
+        else:
+            owner.search = original_search
+
+
+def bounded_image_conversion(native: Callable[[Any], None]) -> Callable[[Any], None]:
+    @wraps(native)
+    def convert(templates: Any) -> None:
+        identifiers = templates.ids or templates.with_context(active_test=False).search([]).ids
+        for offset in range(0, len(identifiers), IMAGE_BATCH_SIZE):
+            selected = identifiers[offset : offset + IMAGE_BATCH_SIZE]
+            with scope_native_template_search(type(templates), selected):
+                native(templates.browse(selected))
+            templates.env.flush_all()
+            templates.env.invalidate_all(flush=False)
+
+    return convert
+
+
+def install_bounded_image_conversion() -> None:
+    from odoo.addons.website_sale.models.product_template import ProductTemplate
+
+    native = getattr(ProductTemplate, "_create_extra_variant_images", None)
+    if not callable(native):
+        raise RuntimeError("The selected Website Sale has no native extra-image conversion")
+    ProductTemplate._create_extra_variant_images = bounded_image_conversion(native)
+
+
+def native_image_copies(cursor: Any, source_maximum: int) -> set[tuple[str, int]]:
+    cursor.execute(
+        "SELECT image.id "
+        "FROM product_image image "
+        "JOIN ir_attachment target ON target.res_model='product.image' AND target.res_id=image.id AND target.res_field='image_1920' "
+        "JOIN LATERAL (SELECT original.id FROM ir_attachment original "
+        "LEFT JOIN product_product variant ON original.res_model='product.product' AND variant.id=original.res_id "
+        "WHERE original.checksum=target.checksum AND original.store_fname=target.store_fname AND ((original.res_model='product.template' "
+        "AND original.res_field='image_1920' AND original.res_id=image.product_tmpl_id) OR "
+        "(original.res_model='product.product' AND original.res_field='image_variant_1920' "
+        "AND variant.product_tmpl_id=image.product_tmpl_id)) ORDER BY original.id LIMIT 1) source ON TRUE "
+        "WHERE image.id>%s ORDER BY image.id",
+        (source_maximum,),
+    )
+    rows = cursor.fetchall()
+    identifiers = {row[0] for row in rows}
+    if len(identifiers) != len(rows):
+        raise ValueError("Native image copies have ambiguous main attachments")
+    return {("product.image", identifier) for identifier in identifiers}

docs/tooling/workspace-cli.md

@@ -691,6 +691,13 @@ Notes
   payloads. A language-keyed label naming an action class is retained as text;
   complete model/record references, unknown encodings and other JSON payloads
   keep their existing refusal behavior.
+  The Website Sale pre-stage calls the native image conversion in bounded groups,
+  retaining native selection and image associations without committing between groups.
+  Attachment content reads memoize file bytes in the ORM raw cache. Each group
+  flushes and invalidates that cache; the first scoped search must name `image_1920`.
+  Original business attachments remain strict. The saved19 driver recognizes new
+  extra-image copies only when database ownership, source checksums and actual bytes
+  match their own template or variant; rollback leaves no separate copy journal.
   Its Mail pre-stage reuses the guarded stock-view conversion from
   [website227](https://github.com/cbusillo/odoo-tenant-cm-website/issues/227).
   Before any shared bridge effect, `mail_view_20.py` locks and compares the

tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py

@@ -13,7 +13,7 @@
 import time
 import uuid
 from collections.abc import Callable, Iterator
-from contextlib import ExitStack, contextmanager
+from contextlib import ExitStack, closing, contextmanager, nullcontext
 from pathlib import Path
 from typing import Any
 from unittest.mock import patch
@@ -51,7 +51,11 @@ def run(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess:
 
 
 def query(database: str, statement: str, params: tuple = ()) -> list[tuple]:
-    with psycopg2.connect(host="database", user="odoo", dbname=database) as connection, connection.cursor() as cursor:
+    with (
+        closing(psycopg2.connect(host="database", user="odoo", dbname=database)) as connection,
+        connection,
+        connection.cursor() as cursor,
+    ):
         cursor.execute(statement, params)
         return cursor.fetchall() if cursor.description else []
 
@@ -68,6 +72,94 @@ def database_command(statement: sql.Composed) -> None:
 
 @tagged("post_install", "-at_install")
 class TestLiveWorkflows(TransactionCase):
+    def test_native_batch_scope_uses_real_recordsets_and_restores_search(self) -> None:
+        conversion = runpy.run_path("/volumes/scripts/product_images_20.py")
+        records = self.env["res.partner"].create(
+            [{"name": "Batch first"}, {"name": "Batch archived", "active": False}, {"name": "Batch last"}]
+        )
+        model = self.env["res.partner"].with_context(active_test=False)
+        observed = []
+        nested = []
+
+        def native(selected: Any) -> None:
+            observed.extend(selected.search([("id", "in", records.ids), ("image_1920", "=", False)]).ids)
+            nested.append(selected.search([("id", "=", records[-1].id)]).ids)
+
+        original = type(model).search
+        with patch.dict(conversion["bounded_image_conversion"].__globals__, {"IMAGE_BATCH_SIZE": 1}):
+            conversion["bounded_image_conversion"](native)(model)
+        self.assertEqual(observed, model.search([("id", "in", records.ids)]).ids)
+        self.assertTrue(all(ids == [records[-1].id] for ids in nested))
+        self.assertIs(type(model).search, original)
+        observed.clear()
+        globals_ = conversion["bounded_image_conversion"].__globals__
+        with patch.dict(globals_, {"IMAGE_BATCH_SIZE": 1, "scope_native_template_search": lambda *_: nullcontext()}):
+            conversion["bounded_image_conversion"](native)(model)
+        self.assertNotEqual(observed, model.search([("id", "in", records.ids)]).ids)
+
+        def fail(_records: Any) -> None:
+            raise RuntimeError("native failure before the first search")
+
+        for local in (False, True):
+            with patch.object(type(model), "search", original) if local else nullcontext():
+                with self.assertRaises(RuntimeError):
+                    conversion["bounded_image_conversion"](fail)(model)
+                self.assertIs(type(model).search, original)
+                self.assertEqual(model.search([("id", "in", records.ids)], order="id").ids, records.ids)
+
+        def changed_native(selected: Any) -> None:
+            selected.search([("id", "in", records.ids)])
+
+        with self.assertRaises(AssertionError):
+            conversion["bounded_image_conversion"](changed_native)(model)
+        self.assertIs(type(model).search, original)
+
+    def test_native_image_copy_ownership_and_shared_file_use_real_sql(self) -> None:
+        check = runpy.run_path("/volumes/scripts/product_images_20.py")["native_image_copies"]
+        with (
+            closing(psycopg2.connect(host="database", user="odoo", dbname="postgres")) as connection,
+            connection,
+            connection.cursor() as cursor,
+        ):
+            checksum = "source image checksum"
+            cursor.execute("CREATE TEMP TABLE product_image(id integer,product_tmpl_id integer)")
+            cursor.execute("CREATE TEMP TABLE product_product(id integer,product_tmpl_id integer)")
+            cursor.execute(
+                "CREATE TEMP TABLE ir_attachment(id integer,res_model text,res_id integer,res_field text,store_fname text,checksum text)"
+            )
+            cursor.execute("INSERT INTO product_image VALUES(22,1)")
+            cursor.execute("INSERT INTO product_product VALUES(2,1)")
+            cursor.executemany(
+                "INSERT INTO ir_attachment VALUES(%s,%s,%s,%s,%s,%s)",
+                [
+                    (1, "product.template", 1, "image_1920", "source", checksum),
+                    (2, "product.image", 22, "image_1920", "source", checksum),
+                ],
+            )
+            self.assertEqual(check(cursor, 21), {("product.image", 22)})
+            self.assertEqual(check(cursor, 22), set())
+            cursor.execute("UPDATE ir_attachment SET res_model='product.product',res_id=2,res_field='image_variant_1920' WHERE id=1")
+            self.assertEqual(check(cursor, 21), {("product.image", 22)})
+            for statement in (
+                "UPDATE product_image SET product_tmpl_id=9",
+                "UPDATE ir_attachment SET res_field='image_1024' WHERE id=1",
+                "UPDATE ir_attachment SET res_field='image_1024' WHERE id=2",
+                "UPDATE ir_attachment SET checksum='different image' WHERE id=2",
+                "UPDATE ir_attachment SET store_fname='different file' WHERE id=2",
+            ):
+                cursor.execute("SAVEPOINT image_copy_case")
+                cursor.execute(statement)
+                self.assertEqual(check(cursor, 21), set())
+                cursor.execute("ROLLBACK TO SAVEPOINT image_copy_case")
+                cursor.execute("RELEASE SAVEPOINT image_copy_case")
+            cursor.execute(
+                "INSERT INTO ir_attachment SELECT 3,res_model,res_id,res_field,store_fname,checksum FROM ir_attachment WHERE id=2"
+            )
+            with self.assertRaises(ValueError):
+                check(cursor, 21)
+            cursor.execute("DELETE FROM ir_attachment WHERE id=3")
+            connection.rollback()
+
     def test_translated_action_labels_preserve_record_reference_guards(self) -> None:
         check = runpy.run_path("/volumes/scripts/prepare_core_references_20.py")["_check_json_references"]
         cursor = self.env.cr
@@ -105,7 +197,7 @@ def test_sale_lead_times_keep_company_values_and_the_diagnostic_transaction(self
         migrate = runpy.run_path("/volumes/scripts/nativeupgrade/sale/20.0.1.0/pre-company-lead-time.py")["migrate"]
         with self.target(empty=True) as runner:
             database_command(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(runner.local.db_name)))
-            with psycopg2.connect(host="database", user="odoo", dbname=runner.local.db_name) as connection:
+            with closing(psycopg2.connect(host="database", user="odoo", dbname=runner.local.db_name)) as connection, connection:
                 with connection.cursor() as cursor:
                     cursor.execute("CREATE TABLE product_template (id integer PRIMARY KEY, sale_delay integer)")
                     cursor.execute(