From 58047381fa687d9b312ddf69f4db4bf4b5838ae2 Mon Sep 17 00:00:00 2001 From: shiny-code-bot <332303085+shiny-code-app[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:23:00 -0500 Subject: [PATCH 1/5] Bound native20 product image conversion and retain copy lineage --- .../20.0.1.0/pre-product-images.py | 14 ++ docker/scripts/product_images_20.py | 86 ++++++++ docs/tooling/workspace-cli.md | 9 + tests/test_product_images_20.py | 202 ++++++++++++++++++ 4 files changed, 311 insertions(+) create mode 100644 docker/scripts/nativeupgrade/website_sale/20.0.1.0/pre-product-images.py create mode 100644 docker/scripts/product_images_20.py create mode 100644 tests/test_product_images_20.py diff --git a/docker/scripts/nativeupgrade/website_sale/20.0.1.0/pre-product-images.py b/docker/scripts/nativeupgrade/website_sale/20.0.1.0/pre-product-images.py new file mode 100644 index 0000000..8152c4a --- /dev/null +++ b/docker/scripts/nativeupgrade/website_sale/20.0.1.0/pre-product-images.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +import runpy +from pathlib import Path +from typing import Any + + +def migrate(cr: Any, version: str | None) -> None: + if not version or version.split(".", 1)[0] != "19": + return + root = Path(__file__).resolve().parents[3] + diagnostics = runpy.run_path(str(root / "migration_diagnostics.py"))["MigrationDiagnostics"]() + install = runpy.run_path(str(root / "product_images_20.py"))["install_bounded_image_conversion"] + diagnostics.check("Website Sale bounded image loader", install, None, cursor=cr) diff --git a/docker/scripts/product_images_20.py b/docker/scripts/product_images_20.py new file mode 100644 index 0000000..0a05d45 --- /dev/null +++ b/docker/scripts/product_images_20.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +IMAGE_BATCH_SIZE = 32 + + +def create_extra_variant_images(templates: Any) -> None: + from odoo import Command + + identifiers = templates.search([("image_1920", "!=", False)]).ids + for offset in range(0, len(identifiers), IMAGE_BATCH_SIZE): + values = [] + origins = [] + for template in templates.browse(identifiers[offset : offset + IMAGE_BATCH_SIZE]): + template_image = template.image_1920 + first_image = template.product_template_image_ids.sorted("sequence")[:1] + add_main = template_image.content != first_image.image_1920.content + main_attributes = templates.env["product.template.attribute.value"] + for product in template.product_variant_ids: + variant_image = product.image_variant_1920 + first_variant_image = product.variant_image_ids.sorted("sequence")[:1] + if add_main and not variant_image and template_image.content == product.image_1920.content: + main_attributes |= product.product_template_attribute_value_ids + if not variant_image or variant_image.content == first_variant_image.image_1920.content: + continue + values.append( + { + "name": product.display_name, + "product_tmpl_id": template.id, + "attribute_value_ids": [Command.set(product.product_template_attribute_value_ids.ids)], + "image_1920": variant_image, + "sequence": first_variant_image.sequence - 1, + } + ) + origins.append(("product.product", product.id, "image_variant_1920", variant_image)) + if add_main: + values.insert( + 0, + { + "name": template.display_name, + "product_tmpl_id": template.id, + "attribute_value_ids": [Command.set(main_attributes.ids)] + if main_attributes and len(template.product_variant_ids) > 1 + else False, + "image_1920": template_image, + "sequence": first_image.sequence - 1, + }, + ) + origins.insert(0, ("product.template", template.id, "image_1920", template_image)) + expected = [ + (model, identifier, field, hashlib.sha1(image.content).hexdigest()) for model, identifier, field, image in origins + ] + created = templates.env["product.image"].with_context(skip_update_main_image=True).create(values) + created_ids = created.ids + templates.env.flush_all() + templates.env.invalidate_all(flush=False) + copies = [] + for image, (model, identifier, field, checksum) in zip( + templates.env["product.image"].browse(created_ids), expected, strict=True + ): + if hashlib.sha1(image.image_1920.content).hexdigest() != checksum: + raise RuntimeError("A native extra image differs from its original product image") + copies.append( + {"image_id": image.id, "source_model": model, "source_id": identifier, "source_field": field, "sha1": checksum} + ) + report = os.environ.get("ODOO_MIGRATION_DIAGNOSTIC_REPORT") + if report and copies: + with Path(report).with_name("product-image-copies.jsonl").open("a") as stream: + stream.writelines(json.dumps(copy) + "\n" for copy in copies) + stream.flush() + os.fsync(stream.fileno()) + templates.env.invalidate_all() + + +def install_bounded_image_conversion() -> None: + from odoo import api + from odoo.addons.website_sale.models.product_template import ProductTemplate + + if not callable(getattr(ProductTemplate, "_create_extra_variant_images", None)): + raise RuntimeError("The selected Website Sale has no native extra-image conversion") + ProductTemplate._create_extra_variant_images = api.model(create_extra_variant_images) diff --git a/docs/tooling/workspace-cli.md b/docs/tooling/workspace-cli.md index 5515007..e5520ac 100644 --- a/docs/tooling/workspace-cli.md +++ b/docs/tooling/workspace-cli.md @@ -691,6 +691,15 @@ Notes payloads. A language-keyed label naming an action class is retained as text; complete model/record references, unknown encodings and other JSON payloads keep their existing refusal behavior. + The Website Sale pre-stage bounds its native template/variant image conversion + in groups and flushes and invalidates caches between groups without committing. + Existing first-extra images are retained, while new copies keep the native + ordering, variant attributes and source image bytes. Each persisted main image + is reread before recording its source ownership and checksum in the private + `product-image-copies.jsonl` beside the diagnostic report. The saved19 driver + independently verifies those rows and bytes before excluding only their derived + attachments from the unchanged-original-business-file comparison. A failed + copy check remains a reported failure, and cannot produce a completed migration. Its Mail pre-stage reuses the guarded stock-view conversion from [website227](https://github.com/cbusillo/odoo-tenant-cm-website/issues/227). Before any shared bridge effect, `mail_view_20.py` locks and compares the diff --git a/tests/test_product_images_20.py b/tests/test_product_images_20.py new file mode 100644 index 0000000..f313620 --- /dev/null +++ b/tests/test_product_images_20.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import json +import os +import runpy +import sys +import tempfile +import unittest +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any +from unittest.mock import Mock, patch + +SCRIPT = Path(__file__).resolve().parents[1] / "docker/scripts/product_images_20.py" +conversion = runpy.run_path(str(SCRIPT)) + + +class Image: + def __init__(self, content: bytes, environment: Environment) -> None: + self.value = content + self.environment = environment + + def __bool__(self) -> bool: + return bool(self.value) + + @property + def content(self) -> bytes: + if self.value: + self.environment.loaded.add(id(self)) + if len(self.environment.loaded) > self.environment.capacity: + raise MemoryError("image content accumulated beyond the batch") + return self.value + + +class Records: + def __init__(self, records: list[Any], empty_image: Image) -> None: + self.records = records + self.empty_image = empty_image + + def __iter__(self) -> Any: + return iter(self.records) + + def __len__(self) -> int: + return len(self.records) + + def __getitem__(self, selection: slice) -> Records: + return Records(self.records[selection], self.empty_image) + + def __or__(self, other: Records) -> Records: + combined = {record.id: record for record in self.records + other.records} + return Records(list(combined.values()), self.empty_image) + + @property + def ids(self) -> list[int]: + return [record.id for record in self.records] + + @property + def image_1920(self) -> Image: + return self.records[0].image_1920 if self.records else self.empty_image + + @property + def sequence(self) -> int: + return self.records[0].sequence if self.records else 0 + + def sorted(self, field: str) -> Records: + return Records(sorted(self.records, key=lambda record: getattr(record, field)), self.empty_image) + + +class Environment: + def __init__(self, count: int, capacity: int) -> None: + self.capacity = capacity + self.loaded: set[int] = set() + self.empty = Image(b"", self) + self.templates = {} + self.images = {} + self.created_values = [] + self.invalidations = 0 + self.flushes = 0 + self.model = Mock(env=self) + self.model.search.side_effect = lambda _domain: Records(list(self.templates.values()), self.empty) + self.model.browse.side_effect = lambda ids: Records([self.templates[identifier] for identifier in ids], self.empty) + self.image_model = Mock() + self.image_model.with_context.return_value = self.image_model + self.image_model.create.side_effect = self.create + self.image_model.browse.side_effect = lambda ids: Records([self.images[identifier] for identifier in ids], self.empty) + for identifier in range(1, count + 1): + image = Image(f"original image {identifier}".encode(), self) + product = SimpleNamespace( + id=identifier, + display_name=f"Variant {identifier}", + image_1920=image, + image_variant_1920=self.empty, + variant_image_ids=Records([], self.empty), + product_template_attribute_value_ids=Records([], self.empty), + ) + self.templates[identifier] = SimpleNamespace( + id=identifier, + display_name=f"Template {identifier}", + image_1920=image, + product_template_image_ids=Records([], self.empty), + product_variant_ids=Records([product], self.empty), + ) + + def __getitem__(self, model: str) -> Any: + return self.image_model if model == "product.image" else Records([], self.empty) + + def create(self, values: list[dict[str, Any]]) -> Records: + records = [] + for value in values: + stored = {**value, "image_1920": value["image_1920"].content} + self.created_values.append(stored) + identifier = len(self.images) + 1 + image = SimpleNamespace(id=identifier, image_1920=Image(stored["image_1920"], self)) + self.images[identifier] = image + records.append(image) + return Records(records, self.empty) + + def flush_all(self) -> None: + self.flushes += 1 + + def invalidate_all(self, *, flush: bool = True) -> None: + if flush: + self.flush_all() + self.loaded.clear() + self.invalidations += 1 + + +class ProductImageConversionTests(unittest.TestCase): + def setUp(self) -> None: + module = ModuleType("odoo") + module.Command = SimpleNamespace(set=lambda ids: (6, 0, ids)) + self.modules = patch.dict(sys.modules, {"odoo": module}) + self.modules.start() + self.addCleanup(self.modules.stop) + + def test_large_catalog_keeps_original_image_bytes_with_bounded_cache(self) -> None: + batch_size = conversion["IMAGE_BATCH_SIZE"] + count = batch_size * 2 + 1 + bounded = Environment(count, batch_size) + with tempfile.TemporaryDirectory() as directory: + report = Path(directory) / "diagnostic.jsonl" + with patch.dict(os.environ, {"ODOO_MIGRATION_DIAGNOSTIC_REPORT": str(report)}): + conversion["create_extra_variant_images"](bounded.model) + copies = [json.loads(line) for line in report.with_name("product-image-copies.jsonl").read_text().splitlines()] + self.assertEqual(len(copies), count) + self.assertEqual({row["source_id"] for row in copies}, set(bounded.templates)) + self.assertEqual(len(bounded.created_values), count) + for row in bounded.created_values: + identifier = row["product_tmpl_id"] + self.assertEqual(row["image_1920"], f"original image {identifier}".encode()) + self.assertEqual(row["attribute_value_ids"], False) + self.assertEqual(row["sequence"], -1) + self.assertGreater(bounded.invalidations, 1) + self.assertGreater(bounded.flushes, 1) + globals_ = conversion["create_extra_variant_images"].__globals__ + with patch.dict(globals_, {"IMAGE_BATCH_SIZE": count}), patch.dict(os.environ, {}, clear=True): + with self.assertRaises(MemoryError): + conversion["create_extra_variant_images"](Environment(count, batch_size).model) + + def test_existing_extra_images_are_not_duplicated_and_variant_attributes_survive(self) -> None: + environment = Environment(2, 20) + first = environment.templates[1] + first.product_template_image_ids = Records([SimpleNamespace(image_1920=first.image_1920, sequence=7)], environment.empty) + second = environment.templates[2] + default_variant = second.product_variant_ids.records[0] + default_variant.product_template_attribute_value_ids = Records([SimpleNamespace(id=11)], environment.empty) + variant_image = Image(b"variant-specific original", environment) + variant = SimpleNamespace( + id=8, + display_name="Blue variant", + image_1920=variant_image, + image_variant_1920=variant_image, + variant_image_ids=Records([], environment.empty), + product_template_attribute_value_ids=Records([SimpleNamespace(id=12)], environment.empty), + ) + second.product_variant_ids = Records([default_variant, variant], environment.empty) + with patch.dict(os.environ, {}, clear=True): + conversion["create_extra_variant_images"](environment.model) + self.assertEqual(len(environment.created_values), 2) + main, extra = environment.created_values + self.assertEqual(main["product_tmpl_id"], second.id) + self.assertEqual(main["attribute_value_ids"], [(6, 0, [11])]) + self.assertEqual(extra["attribute_value_ids"], [(6, 0, [12])]) + self.assertEqual(extra["image_1920"], b"variant-specific original") + environment.image_model.with_context.assert_called_with(skip_update_main_image=True) + + def test_changed_persisted_copy_fails_without_publishing_preservation_evidence(self) -> None: + environment = Environment(1, 20) + create = environment.create + + def corrupt(values: list[dict[str, Any]]) -> Records: + records = create(values) + records.records[0].image_1920 = Image(b"changed image", environment) + return records + + environment.image_model.create.side_effect = corrupt + with tempfile.TemporaryDirectory() as directory: + report = Path(directory) / "diagnostic.jsonl" + with patch.dict(os.environ, {"ODOO_MIGRATION_DIAGNOSTIC_REPORT": str(report)}): + with self.assertRaises(RuntimeError): + conversion["create_extra_variant_images"](environment.model) + self.assertFalse(report.with_name("product-image-copies.jsonl").exists()) From cd1540cfb5270b98d746860712eea63af35f43b4 Mon Sep 17 00:00:00 2001 From: shiny-code-bot <332303085+shiny-code-app[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:57:59 -0500 Subject: [PATCH 2/5] Delegate image batches to native code and derive copies transactionally --- .../20.0.1.0/pre-product-images.py | 2 +- docker/scripts/product_images_20.py | 139 +++++----- docs/tooling/workspace-cli.md | 14 +- .../tests/test_live_workflows.py | 73 +++++ tests/test_product_images_20.py | 261 ++++++------------ 5 files changed, 236 insertions(+), 253 deletions(-) diff --git a/docker/scripts/nativeupgrade/website_sale/20.0.1.0/pre-product-images.py b/docker/scripts/nativeupgrade/website_sale/20.0.1.0/pre-product-images.py index 8152c4a..b041001 100644 --- a/docker/scripts/nativeupgrade/website_sale/20.0.1.0/pre-product-images.py +++ b/docker/scripts/nativeupgrade/website_sale/20.0.1.0/pre-product-images.py @@ -11,4 +11,4 @@ def migrate(cr: Any, version: str | None) -> None: root = Path(__file__).resolve().parents[3] diagnostics = runpy.run_path(str(root / "migration_diagnostics.py"))["MigrationDiagnostics"]() install = runpy.run_path(str(root / "product_images_20.py"))["install_bounded_image_conversion"] - diagnostics.check("Website Sale bounded image loader", install, None, cursor=cr) + diagnostics.check("Website Sale bounded image loader", install, None) diff --git a/docker/scripts/product_images_20.py b/docker/scripts/product_images_20.py index 0a05d45..283fe46 100644 --- a/docker/scripts/product_images_20.py +++ b/docker/scripts/product_images_20.py @@ -1,86 +1,85 @@ from __future__ import annotations import hashlib -import json -import os +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from functools import wraps from pathlib import Path from typing import Any IMAGE_BATCH_SIZE = 32 -def create_extra_variant_images(templates: Any) -> None: - from odoo import Command +@contextmanager +def scope_native_template_search(owner: Any, selected: list[int]) -> Iterator[None]: + from odoo.fields import Domain - identifiers = templates.search([("image_1920", "!=", False)]).ids - for offset in range(0, len(identifiers), IMAGE_BATCH_SIZE): - values = [] - origins = [] - for template in templates.browse(identifiers[offset : offset + IMAGE_BATCH_SIZE]): - template_image = template.image_1920 - first_image = template.product_template_image_ids.sorted("sequence")[:1] - add_main = template_image.content != first_image.image_1920.content - main_attributes = templates.env["product.template.attribute.value"] - for product in template.product_variant_ids: - variant_image = product.image_variant_1920 - first_variant_image = product.variant_image_ids.sorted("sequence")[:1] - if add_main and not variant_image and template_image.content == product.image_1920.content: - main_attributes |= product.product_template_attribute_value_ids - if not variant_image or variant_image.content == first_variant_image.image_1920.content: - continue - values.append( - { - "name": product.display_name, - "product_tmpl_id": template.id, - "attribute_value_ids": [Command.set(product.product_template_attribute_value_ids.ids)], - "image_1920": variant_image, - "sequence": first_variant_image.sequence - 1, - } - ) - origins.append(("product.product", product.id, "image_variant_1920", variant_image)) - if add_main: - values.insert( - 0, - { - "name": template.display_name, - "product_tmpl_id": template.id, - "attribute_value_ids": [Command.set(main_attributes.ids)] - if main_attributes and len(template.product_variant_ids) > 1 - else False, - "image_1920": template_image, - "sequence": first_image.sequence - 1, - }, - ) - origins.insert(0, ("product.template", template.id, "image_1920", template_image)) - expected = [ - (model, identifier, field, hashlib.sha1(image.content).hexdigest()) for model, identifier, field, image in origins - ] - created = templates.env["product.image"].with_context(skip_update_main_image=True).create(values) - created_ids = created.ids - templates.env.flush_all() - templates.env.invalidate_all(flush=False) - copies = [] - for image, (model, identifier, field, checksum) in zip( - templates.env["product.image"].browse(created_ids), expected, strict=True - ): - if hashlib.sha1(image.image_1920.content).hexdigest() != checksum: - raise RuntimeError("A native extra image differs from its original product image") - copies.append( - {"image_id": image.id, "source_model": model, "source_id": identifier, "source_field": field, "sha1": checksum} - ) - report = os.environ.get("ODOO_MIGRATION_DIAGNOSTIC_REPORT") - if report and copies: - with Path(report).with_name("product-image-copies.jsonl").open("a") as stream: - stream.writelines(json.dumps(copy) + "\n" for copy in copies) - stream.flush() - os.fsync(stream.fileno()) - templates.env.invalidate_all() + original_search = owner.search + inherited = "search" not in owner.__dict__ + first_search = True + + @wraps(original_search) + def search(records: Any, domain: Any, *args: Any, **kwargs: Any) -> Any: + nonlocal first_search + if first_search: + first_search = False + domain = Domain(domain) & Domain("id", "in", selected) + return original_search(records, domain, *args, **kwargs) + + owner.search = search + try: + yield + finally: + if inherited: + del owner.search + else: + owner.search = original_search + + +def bounded_image_conversion(native: Callable[[Any], None]) -> Callable[[Any], None]: + @wraps(native) + def convert(templates: Any) -> None: + identifiers = templates.ids or templates.with_context(active_test=False).search([]).ids + for offset in range(0, len(identifiers), IMAGE_BATCH_SIZE): + selected = identifiers[offset : offset + IMAGE_BATCH_SIZE] + with scope_native_template_search(type(templates), selected): + native(templates.browse(selected)) + templates.env.flush_all() + templates.env.invalidate_all(flush=False) + + return convert def install_bounded_image_conversion() -> None: - from odoo import api from odoo.addons.website_sale.models.product_template import ProductTemplate - if not callable(getattr(ProductTemplate, "_create_extra_variant_images", None)): + native = getattr(ProductTemplate, "_create_extra_variant_images", None) + if not callable(native): raise RuntimeError("The selected Website Sale has no native extra-image conversion") - ProductTemplate._create_extra_variant_images = api.model(create_extra_variant_images) + ProductTemplate._create_extra_variant_images = bounded_image_conversion(native) + + +def native_image_copies(cursor: Any, store: Path, source_maximum: int) -> set[tuple[str, int]]: + cursor.execute( + "SELECT image.id,target.store_fname,target.checksum,source.store_fname " + "FROM product_image image " + "JOIN ir_attachment target ON target.res_model='product.image' AND target.res_id=image.id AND target.res_field='image_1920' " + "JOIN LATERAL (SELECT original.store_fname FROM ir_attachment original " + "LEFT JOIN product_product variant ON original.res_model='product.product' AND variant.id=original.res_id " + "WHERE original.checksum=target.checksum AND ((original.res_model='product.template' " + "AND original.res_field='image_1920' AND original.res_id=image.product_tmpl_id) OR " + "(original.res_model='product.product' AND original.res_field='image_variant_1920' " + "AND variant.product_tmpl_id=image.product_tmpl_id)) ORDER BY original.id LIMIT 1) source ON TRUE " + "WHERE image.id>%s ORDER BY image.id", + (source_maximum,), + ) + rows = cursor.fetchall() + identifiers = {row[0] for row in rows} + if len(identifiers) != len(rows): + raise ValueError("Native image copies have ambiguous main attachments") + for _identifier, target, checksum, original in rows: + for name in {target, original}: + with (store / name).open("rb") as stream: + if hashlib.file_digest(stream, "sha1").hexdigest() != checksum: + raise ValueError("Native image-copy bytes differ from their original product images") + return {("product.image", identifier) for identifier in identifiers} diff --git a/docs/tooling/workspace-cli.md b/docs/tooling/workspace-cli.md index e5520ac..4f7c1cf 100644 --- a/docs/tooling/workspace-cli.md +++ b/docs/tooling/workspace-cli.md @@ -691,15 +691,11 @@ Notes payloads. A language-keyed label naming an action class is retained as text; complete model/record references, unknown encodings and other JSON payloads keep their existing refusal behavior. - The Website Sale pre-stage bounds its native template/variant image conversion - in groups and flushes and invalidates caches between groups without committing. - Existing first-extra images are retained, while new copies keep the native - ordering, variant attributes and source image bytes. Each persisted main image - is reread before recording its source ownership and checksum in the private - `product-image-copies.jsonl` beside the diagnostic report. The saved19 driver - independently verifies those rows and bytes before excluding only their derived - attachments from the unchanged-original-business-file comparison. A failed - copy check remains a reported failure, and cannot produce a completed migration. + The Website Sale pre-stage calls the native image conversion in bounded groups, + retaining native selection and image associations without committing between groups. + Original business attachments remain strict. The saved19 driver recognizes new + extra-image copies only when database ownership, source checksums and actual bytes + match their own template or variant; rollback leaves no separate copy journal. Its Mail pre-stage reuses the guarded stock-view conversion from [website227](https://github.com/cbusillo/odoo-tenant-cm-website/issues/227). Before any shared bridge effect, `mail_view_20.py` locks and compares the diff --git a/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py b/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py index 463e818..d0f152d 100644 --- a/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py +++ b/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py @@ -4,6 +4,7 @@ import base64 import configparser +import hashlib import json import os import runpy @@ -68,6 +69,78 @@ def database_command(statement: sql.Composed) -> None: @tagged("post_install", "-at_install") class TestLiveWorkflows(TransactionCase): + def test_native_batch_scope_uses_real_recordsets_and_restores_search(self) -> None: + conversion = runpy.run_path("/volumes/scripts/product_images_20.py") + records = self.env["res.partner"].create( + [{"name": "Batch first"}, {"name": "Batch archived", "active": False}, {"name": "Batch last"}] + ) + model = self.env["res.partner"].with_context(active_test=False) + observed = [] + nested = [] + + def native(selected: Any) -> None: + observed.extend(selected.search([("id", "in", records.ids)], order="id").ids) + nested.append(selected.search([("id", "=", records[-1].id)]).ids) + + original = type(model).search + with patch.dict(conversion["bounded_image_conversion"].__globals__, {"IMAGE_BATCH_SIZE": 1}): + conversion["bounded_image_conversion"](native)(model) + self.assertEqual(observed, records.ids) + self.assertTrue(all(ids == [records[-1].id] for ids in nested)) + self.assertIs(type(model).search, original) + + def test_native_image_copy_ownership_and_bytes_use_real_sql(self) -> None: + check = runpy.run_path("/volumes/scripts/product_images_20.py")["native_image_copies"] + with ( + psycopg2.connect(host="database", user="odoo", dbname="postgres") as connection, + connection.cursor() as cursor, + tempfile.TemporaryDirectory() as directory, + ): + store = Path(directory) + payload = b"original product image" + checksum = hashlib.sha1(payload).hexdigest() + for name in ("source", "copy"): + (store / name).write_bytes(payload) + cursor.execute("CREATE TEMP TABLE product_image(id integer,product_tmpl_id integer)") + cursor.execute("CREATE TEMP TABLE product_product(id integer,product_tmpl_id integer)") + cursor.execute( + "CREATE TEMP TABLE ir_attachment(id integer,res_model text,res_id integer,res_field text,store_fname text,checksum text)" + ) + cursor.execute("INSERT INTO product_image VALUES(22,1)") + cursor.execute("INSERT INTO product_product VALUES(2,1)") + cursor.executemany( + "INSERT INTO ir_attachment VALUES(%s,%s,%s,%s,%s,%s)", + [ + (1, "product.template", 1, "image_1920", "source", checksum), + (2, "product.image", 22, "image_1920", "copy", checksum), + ], + ) + self.assertEqual(check(cursor, store, 21), {("product.image", 22)}) + self.assertEqual(check(cursor, store, 22), set()) + cursor.execute("UPDATE ir_attachment SET res_model='product.product',res_id=2,res_field='image_variant_1920' WHERE id=1") + self.assertEqual(check(cursor, store, 21), {("product.image", 22)}) + for statement in ( + "UPDATE product_image SET product_tmpl_id=9", + "UPDATE ir_attachment SET res_field='image_1024' WHERE id=1", + "UPDATE ir_attachment SET res_field='image_1024' WHERE id=2", + "UPDATE ir_attachment SET checksum='different image' WHERE id=2", + ): + cursor.execute("SAVEPOINT image_copy_case") + cursor.execute(statement) + self.assertEqual(check(cursor, store, 21), set()) + cursor.execute("ROLLBACK TO SAVEPOINT image_copy_case") + cursor.execute("RELEASE SAVEPOINT image_copy_case") + cursor.execute( + "INSERT INTO ir_attachment SELECT 3,res_model,res_id,res_field,store_fname,checksum FROM ir_attachment WHERE id=2" + ) + with self.assertRaises(ValueError): + check(cursor, store, 21) + cursor.execute("DELETE FROM ir_attachment WHERE id=3") + (store / "copy").write_bytes(b"corrupt image bytes") + with self.assertRaises(ValueError): + check(cursor, store, 21) + connection.rollback() + def test_translated_action_labels_preserve_record_reference_guards(self) -> None: check = runpy.run_path("/volumes/scripts/prepare_core_references_20.py")["_check_json_references"] cursor = self.env.cr diff --git a/tests/test_product_images_20.py b/tests/test_product_images_20.py index f313620..9319e2c 100644 --- a/tests/test_product_images_20.py +++ b/tests/test_product_images_20.py @@ -1,202 +1,117 @@ from __future__ import annotations -import json -import os import runpy import sys -import tempfile import unittest +from contextlib import nullcontext from pathlib import Path -from types import ModuleType, SimpleNamespace +from types import ModuleType from typing import Any -from unittest.mock import Mock, patch +from unittest.mock import patch -SCRIPT = Path(__file__).resolve().parents[1] / "docker/scripts/product_images_20.py" -conversion = runpy.run_path(str(SCRIPT)) +conversion = runpy.run_path(str(Path(__file__).resolve().parents[1] / "docker/scripts/product_images_20.py")) -class Image: - def __init__(self, content: bytes, environment: Environment) -> None: - self.value = content - self.environment = environment +class Domain(list): + def __init__(self, *values: Any) -> None: + super().__init__(values[0] if len(values) == 1 else [values]) - def __bool__(self) -> bool: - return bool(self.value) + def __and__(self, other: Domain) -> Domain: + return Domain([*self, *other]) - @property - def content(self) -> bytes: - if self.value: - self.environment.loaded.add(id(self)) - if len(self.environment.loaded) > self.environment.capacity: - raise MemoryError("image content accumulated beyond the batch") - return self.value +class Environment: + def __init__(self, capacity: int) -> None: + self.capacity = capacity + self.loaded: set[int] = set() -class Records: - def __init__(self, records: list[Any], empty_image: Image) -> None: - self.records = records - self.empty_image = empty_image - - def __iter__(self) -> Any: - return iter(self.records) - - def __len__(self) -> int: - return len(self.records) - - def __getitem__(self, selection: slice) -> Records: - return Records(self.records[selection], self.empty_image) - - def __or__(self, other: Records) -> Records: - combined = {record.id: record for record in self.records + other.records} - return Records(list(combined.values()), self.empty_image) - - @property - def ids(self) -> list[int]: - return [record.id for record in self.records] + def flush_all(self) -> None: + pass - @property - def image_1920(self) -> Image: - return self.records[0].image_1920 if self.records else self.empty_image + def invalidate_all(self, *, flush: bool) -> None: + self.loaded.clear() - @property - def sequence(self) -> int: - return self.records[0].sequence if self.records else 0 - def sorted(self, field: str) -> Records: - return Records(sorted(self.records, key=lambda record: getattr(record, field)), self.empty_image) +class Model: + rows: list[dict[str, Any]] = [] + def __init__(self, environment: Environment, ids: list[int] | None = None, *, active_test: bool = True) -> None: + self.env = environment + self.ids = ids or [] + self.active_test = active_test -class Environment: - def __init__(self, count: int, capacity: int) -> None: - self.capacity = capacity - self.loaded: set[int] = set() - self.empty = Image(b"", self) - self.templates = {} - self.images = {} - self.created_values = [] - self.invalidations = 0 - self.flushes = 0 - self.model = Mock(env=self) - self.model.search.side_effect = lambda _domain: Records(list(self.templates.values()), self.empty) - self.model.browse.side_effect = lambda ids: Records([self.templates[identifier] for identifier in ids], self.empty) - self.image_model = Mock() - self.image_model.with_context.return_value = self.image_model - self.image_model.create.side_effect = self.create - self.image_model.browse.side_effect = lambda ids: Records([self.images[identifier] for identifier in ids], self.empty) - for identifier in range(1, count + 1): - image = Image(f"original image {identifier}".encode(), self) - product = SimpleNamespace( - id=identifier, - display_name=f"Variant {identifier}", - image_1920=image, - image_variant_1920=self.empty, - variant_image_ids=Records([], self.empty), - product_template_attribute_value_ids=Records([], self.empty), - ) - self.templates[identifier] = SimpleNamespace( - id=identifier, - display_name=f"Template {identifier}", - image_1920=image, - product_template_image_ids=Records([], self.empty), - product_variant_ids=Records([product], self.empty), - ) - - def __getitem__(self, model: str) -> Any: - return self.image_model if model == "product.image" else Records([], self.empty) - - def create(self, values: list[dict[str, Any]]) -> Records: - records = [] - for value in values: - stored = {**value, "image_1920": value["image_1920"].content} - self.created_values.append(stored) - identifier = len(self.images) + 1 - image = SimpleNamespace(id=identifier, image_1920=Image(stored["image_1920"], self)) - self.images[identifier] = image - records.append(image) - return Records(records, self.empty) + def browse(self, ids: list[int]) -> Model: + return type(self)(self.env, ids, active_test=self.active_test) - def flush_all(self) -> None: - self.flushes += 1 + def with_context(self, *, active_test: bool) -> Model: + return type(self)(self.env, self.ids, active_test=active_test) - def invalidate_all(self, *, flush: bool = True) -> None: - if flush: - self.flush_all() - self.loaded.clear() - self.invalidations += 1 + def search(self, domain: list[tuple]) -> Model: + selected = [] + for row in self.rows: + if self.active_test and not row["active"]: + continue + if all(row[field] in value if operator == "in" else row[field] == value for field, operator, value in domain): + selected.append(row["id"]) + return self.browse(selected) class ProductImageConversionTests(unittest.TestCase): def setUp(self) -> None: + fields = ModuleType("odoo.fields") + fields.Domain = Domain module = ModuleType("odoo") - module.Command = SimpleNamespace(set=lambda ids: (6, 0, ids)) - self.modules = patch.dict(sys.modules, {"odoo": module}) + module.fields = fields + self.modules = patch.dict(sys.modules, {"odoo": module, "odoo.fields": fields}) self.modules.start() self.addCleanup(self.modules.stop) - - def test_large_catalog_keeps_original_image_bytes_with_bounded_cache(self) -> None: - batch_size = conversion["IMAGE_BATCH_SIZE"] - count = batch_size * 2 + 1 - bounded = Environment(count, batch_size) - with tempfile.TemporaryDirectory() as directory: - report = Path(directory) / "diagnostic.jsonl" - with patch.dict(os.environ, {"ODOO_MIGRATION_DIAGNOSTIC_REPORT": str(report)}): - conversion["create_extra_variant_images"](bounded.model) - copies = [json.loads(line) for line in report.with_name("product-image-copies.jsonl").read_text().splitlines()] - self.assertEqual(len(copies), count) - self.assertEqual({row["source_id"] for row in copies}, set(bounded.templates)) - self.assertEqual(len(bounded.created_values), count) - for row in bounded.created_values: - identifier = row["product_tmpl_id"] - self.assertEqual(row["image_1920"], f"original image {identifier}".encode()) - self.assertEqual(row["attribute_value_ids"], False) - self.assertEqual(row["sequence"], -1) - self.assertGreater(bounded.invalidations, 1) - self.assertGreater(bounded.flushes, 1) - globals_ = conversion["create_extra_variant_images"].__globals__ - with patch.dict(globals_, {"IMAGE_BATCH_SIZE": count}), patch.dict(os.environ, {}, clear=True): + self.capacity = conversion["IMAGE_BATCH_SIZE"] + self.count = self.capacity * 2 + 1 + Model.rows = [ + {"id": identifier, "active": identifier != 3, "eligible": identifier != 7} for identifier in range(1, self.count + 1) + ] + + def test_native_selection_is_bounded_without_restricting_nested_searches(self) -> None: + seen = [] + nested = [] + + def native(model: Model) -> None: + selected = model.search([("eligible", "=", True)]).ids + model.env.loaded.update(selected) + if len(model.env.loaded) > model.env.capacity: + raise MemoryError("native image cache exceeded its capacity") + seen.extend(selected) + nested.append(model.search([("id", "=", self.count)]).ids) + + bounded = conversion["bounded_image_conversion"](native) + bounded(Model(Environment(self.capacity))) + expected = [row["id"] for row in Model.rows if row["active"] and row["eligible"]] + self.assertEqual(seen, expected) + self.assertTrue(all(ids == [self.count] for ids in nested)) + globals_ = conversion["bounded_image_conversion"].__globals__ + with patch.dict(globals_, {"scope_native_template_search": lambda *_: nullcontext()}): with self.assertRaises(MemoryError): - conversion["create_extra_variant_images"](Environment(count, batch_size).model) - - def test_existing_extra_images_are_not_duplicated_and_variant_attributes_survive(self) -> None: - environment = Environment(2, 20) - first = environment.templates[1] - first.product_template_image_ids = Records([SimpleNamespace(image_1920=first.image_1920, sequence=7)], environment.empty) - second = environment.templates[2] - default_variant = second.product_variant_ids.records[0] - default_variant.product_template_attribute_value_ids = Records([SimpleNamespace(id=11)], environment.empty) - variant_image = Image(b"variant-specific original", environment) - variant = SimpleNamespace( - id=8, - display_name="Blue variant", - image_1920=variant_image, - image_variant_1920=variant_image, - variant_image_ids=Records([], environment.empty), - product_template_attribute_value_ids=Records([SimpleNamespace(id=12)], environment.empty), - ) - second.product_variant_ids = Records([default_variant, variant], environment.empty) - with patch.dict(os.environ, {}, clear=True): - conversion["create_extra_variant_images"](environment.model) - self.assertEqual(len(environment.created_values), 2) - main, extra = environment.created_values - self.assertEqual(main["product_tmpl_id"], second.id) - self.assertEqual(main["attribute_value_ids"], [(6, 0, [11])]) - self.assertEqual(extra["attribute_value_ids"], [(6, 0, [12])]) - self.assertEqual(extra["image_1920"], b"variant-specific original") - environment.image_model.with_context.assert_called_with(skip_update_main_image=True) - - def test_changed_persisted_copy_fails_without_publishing_preservation_evidence(self) -> None: - environment = Environment(1, 20) - create = environment.create - - def corrupt(values: list[dict[str, Any]]) -> Records: - records = create(values) - records.records[0].image_1920 = Image(b"changed image", environment) - return records - - environment.image_model.create.side_effect = corrupt - with tempfile.TemporaryDirectory() as directory: - report = Path(directory) / "diagnostic.jsonl" - with patch.dict(os.environ, {"ODOO_MIGRATION_DIAGNOSTIC_REPORT": str(report)}): - with self.assertRaises(RuntimeError): - conversion["create_extra_variant_images"](environment.model) - self.assertFalse(report.with_name("product-image-copies.jsonl").exists()) + bounded(Model(Environment(self.capacity))) + + def test_native_inactive_context_and_explicit_record_subset_survive(self) -> None: + seen = [] + bounded = conversion["bounded_image_conversion"](lambda model: seen.extend(model.search([("eligible", "=", True)]).ids)) + bounded(Model(Environment(self.capacity), active_test=False)) + self.assertEqual(seen, [row["id"] for row in Model.rows if row["eligible"]]) + seen.clear() + bounded(Model(Environment(self.capacity), [2, 3, 7], active_test=False)) + self.assertEqual(seen, [2, 3]) + + def test_native_failure_restores_local_and_inherited_search_methods(self) -> None: + class Inherited(Model): + pass + + def fail(_model: Model) -> None: + raise RuntimeError("native failure before its first search") + + for owner in (Model, Inherited): + original = owner.search + with self.assertRaises(RuntimeError): + conversion["bounded_image_conversion"](fail)(owner(Environment(self.capacity))) + self.assertIs(owner.search, original) + self.assertEqual(owner(Environment(self.capacity)).search([]).ids, [row["id"] for row in Model.rows if row["active"]]) From 3b9be52082b7cf900a4ec14f6d5d401448c66108 Mon Sep 17 00:00:00 2001 From: shiny-code-bot <332303085+shiny-code-app[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 12:22:36 -0500 Subject: [PATCH 3/5] Share original-file byte proof and keep image regressions on real models --- docker/scripts/product_images_20.py | 15 +-- .../tests/test_live_workflows.py | 54 ++++---- tests/test_product_images_20.py | 117 ------------------ 3 files changed, 37 insertions(+), 149 deletions(-) delete mode 100644 tests/test_product_images_20.py diff --git a/docker/scripts/product_images_20.py b/docker/scripts/product_images_20.py index 283fe46..75831c8 100644 --- a/docker/scripts/product_images_20.py +++ b/docker/scripts/product_images_20.py @@ -1,10 +1,8 @@ from __future__ import annotations -import hashlib from collections.abc import Callable, Iterator from contextlib import contextmanager from functools import wraps -from pathlib import Path from typing import Any IMAGE_BATCH_SIZE = 32 @@ -59,14 +57,14 @@ def install_bounded_image_conversion() -> None: ProductTemplate._create_extra_variant_images = bounded_image_conversion(native) -def native_image_copies(cursor: Any, store: Path, source_maximum: int) -> set[tuple[str, int]]: +def native_image_copies(cursor: Any, source_maximum: int) -> set[tuple[str, int]]: cursor.execute( - "SELECT image.id,target.store_fname,target.checksum,source.store_fname " + "SELECT image.id " "FROM product_image image " "JOIN ir_attachment target ON target.res_model='product.image' AND target.res_id=image.id AND target.res_field='image_1920' " - "JOIN LATERAL (SELECT original.store_fname FROM ir_attachment original " + "JOIN LATERAL (SELECT original.id FROM ir_attachment original " "LEFT JOIN product_product variant ON original.res_model='product.product' AND variant.id=original.res_id " - "WHERE original.checksum=target.checksum AND ((original.res_model='product.template' " + "WHERE original.checksum=target.checksum AND original.store_fname=target.store_fname AND ((original.res_model='product.template' " "AND original.res_field='image_1920' AND original.res_id=image.product_tmpl_id) OR " "(original.res_model='product.product' AND original.res_field='image_variant_1920' " "AND variant.product_tmpl_id=image.product_tmpl_id)) ORDER BY original.id LIMIT 1) source ON TRUE " @@ -77,9 +75,4 @@ def native_image_copies(cursor: Any, store: Path, source_maximum: int) -> set[tu identifiers = {row[0] for row in rows} if len(identifiers) != len(rows): raise ValueError("Native image copies have ambiguous main attachments") - for _identifier, target, checksum, original in rows: - for name in {target, original}: - with (store / name).open("rb") as stream: - if hashlib.file_digest(stream, "sha1").hexdigest() != checksum: - raise ValueError("Native image-copy bytes differ from their original product images") return {("product.image", identifier) for identifier in identifiers} diff --git a/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py b/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py index d0f152d..cdf3a88 100644 --- a/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py +++ b/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py @@ -4,7 +4,6 @@ import base64 import configparser -import hashlib import json import os import runpy @@ -14,7 +13,7 @@ import time import uuid from collections.abc import Callable, Iterator -from contextlib import ExitStack, contextmanager +from contextlib import ExitStack, closing, contextmanager, nullcontext from pathlib import Path from typing import Any from unittest.mock import patch @@ -52,7 +51,11 @@ def run(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess: def query(database: str, statement: str, params: tuple = ()) -> list[tuple]: - with psycopg2.connect(host="database", user="odoo", dbname=database) as connection, connection.cursor() as cursor: + with ( + closing(psycopg2.connect(host="database", user="odoo", dbname=database)) as connection, + connection, + connection.cursor() as cursor, + ): cursor.execute(statement, params) return cursor.fetchall() if cursor.description else [] @@ -88,19 +91,30 @@ def native(selected: Any) -> None: self.assertEqual(observed, records.ids) self.assertTrue(all(ids == [records[-1].id] for ids in nested)) self.assertIs(type(model).search, original) + observed.clear() + globals_ = conversion["bounded_image_conversion"].__globals__ + with patch.dict(globals_, {"IMAGE_BATCH_SIZE": 1, "scope_native_template_search": lambda *_: nullcontext()}): + conversion["bounded_image_conversion"](native)(model) + self.assertNotEqual(observed, records.ids) + + def fail(_records: Any) -> None: + raise RuntimeError("native failure before the first search") - def test_native_image_copy_ownership_and_bytes_use_real_sql(self) -> None: + for local in (False, True): + with patch.object(type(model), "search", original) if local else nullcontext(): + with self.assertRaises(RuntimeError): + conversion["bounded_image_conversion"](fail)(model) + self.assertIs(type(model).search, original) + self.assertEqual(model.search([("id", "in", records.ids)], order="id").ids, records.ids) + + def test_native_image_copy_ownership_and_shared_file_use_real_sql(self) -> None: check = runpy.run_path("/volumes/scripts/product_images_20.py")["native_image_copies"] with ( - psycopg2.connect(host="database", user="odoo", dbname="postgres") as connection, + closing(psycopg2.connect(host="database", user="odoo", dbname="postgres")) as connection, + connection, connection.cursor() as cursor, - tempfile.TemporaryDirectory() as directory, ): - store = Path(directory) - payload = b"original product image" - checksum = hashlib.sha1(payload).hexdigest() - for name in ("source", "copy"): - (store / name).write_bytes(payload) + checksum = "source image checksum" cursor.execute("CREATE TEMP TABLE product_image(id integer,product_tmpl_id integer)") cursor.execute("CREATE TEMP TABLE product_product(id integer,product_tmpl_id integer)") cursor.execute( @@ -112,33 +126,31 @@ def test_native_image_copy_ownership_and_bytes_use_real_sql(self) -> None: "INSERT INTO ir_attachment VALUES(%s,%s,%s,%s,%s,%s)", [ (1, "product.template", 1, "image_1920", "source", checksum), - (2, "product.image", 22, "image_1920", "copy", checksum), + (2, "product.image", 22, "image_1920", "source", checksum), ], ) - self.assertEqual(check(cursor, store, 21), {("product.image", 22)}) - self.assertEqual(check(cursor, store, 22), set()) + self.assertEqual(check(cursor, 21), {("product.image", 22)}) + self.assertEqual(check(cursor, 22), set()) cursor.execute("UPDATE ir_attachment SET res_model='product.product',res_id=2,res_field='image_variant_1920' WHERE id=1") - self.assertEqual(check(cursor, store, 21), {("product.image", 22)}) + self.assertEqual(check(cursor, 21), {("product.image", 22)}) for statement in ( "UPDATE product_image SET product_tmpl_id=9", "UPDATE ir_attachment SET res_field='image_1024' WHERE id=1", "UPDATE ir_attachment SET res_field='image_1024' WHERE id=2", "UPDATE ir_attachment SET checksum='different image' WHERE id=2", + "UPDATE ir_attachment SET store_fname='different file' WHERE id=2", ): cursor.execute("SAVEPOINT image_copy_case") cursor.execute(statement) - self.assertEqual(check(cursor, store, 21), set()) + self.assertEqual(check(cursor, 21), set()) cursor.execute("ROLLBACK TO SAVEPOINT image_copy_case") cursor.execute("RELEASE SAVEPOINT image_copy_case") cursor.execute( "INSERT INTO ir_attachment SELECT 3,res_model,res_id,res_field,store_fname,checksum FROM ir_attachment WHERE id=2" ) with self.assertRaises(ValueError): - check(cursor, store, 21) + check(cursor, 21) cursor.execute("DELETE FROM ir_attachment WHERE id=3") - (store / "copy").write_bytes(b"corrupt image bytes") - with self.assertRaises(ValueError): - check(cursor, store, 21) connection.rollback() def test_translated_action_labels_preserve_record_reference_guards(self) -> None: @@ -178,7 +190,7 @@ def test_sale_lead_times_keep_company_values_and_the_diagnostic_transaction(self migrate = runpy.run_path("/volumes/scripts/nativeupgrade/sale/20.0.1.0/pre-company-lead-time.py")["migrate"] with self.target(empty=True) as runner: database_command(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(runner.local.db_name))) - with psycopg2.connect(host="database", user="odoo", dbname=runner.local.db_name) as connection: + with closing(psycopg2.connect(host="database", user="odoo", dbname=runner.local.db_name)) as connection, connection: with connection.cursor() as cursor: cursor.execute("CREATE TABLE product_template (id integer PRIMARY KEY, sale_delay integer)") cursor.execute( diff --git a/tests/test_product_images_20.py b/tests/test_product_images_20.py deleted file mode 100644 index 9319e2c..0000000 --- a/tests/test_product_images_20.py +++ /dev/null @@ -1,117 +0,0 @@ -from __future__ import annotations - -import runpy -import sys -import unittest -from contextlib import nullcontext -from pathlib import Path -from types import ModuleType -from typing import Any -from unittest.mock import patch - -conversion = runpy.run_path(str(Path(__file__).resolve().parents[1] / "docker/scripts/product_images_20.py")) - - -class Domain(list): - def __init__(self, *values: Any) -> None: - super().__init__(values[0] if len(values) == 1 else [values]) - - def __and__(self, other: Domain) -> Domain: - return Domain([*self, *other]) - - -class Environment: - def __init__(self, capacity: int) -> None: - self.capacity = capacity - self.loaded: set[int] = set() - - def flush_all(self) -> None: - pass - - def invalidate_all(self, *, flush: bool) -> None: - self.loaded.clear() - - -class Model: - rows: list[dict[str, Any]] = [] - - def __init__(self, environment: Environment, ids: list[int] | None = None, *, active_test: bool = True) -> None: - self.env = environment - self.ids = ids or [] - self.active_test = active_test - - def browse(self, ids: list[int]) -> Model: - return type(self)(self.env, ids, active_test=self.active_test) - - def with_context(self, *, active_test: bool) -> Model: - return type(self)(self.env, self.ids, active_test=active_test) - - def search(self, domain: list[tuple]) -> Model: - selected = [] - for row in self.rows: - if self.active_test and not row["active"]: - continue - if all(row[field] in value if operator == "in" else row[field] == value for field, operator, value in domain): - selected.append(row["id"]) - return self.browse(selected) - - -class ProductImageConversionTests(unittest.TestCase): - def setUp(self) -> None: - fields = ModuleType("odoo.fields") - fields.Domain = Domain - module = ModuleType("odoo") - module.fields = fields - self.modules = patch.dict(sys.modules, {"odoo": module, "odoo.fields": fields}) - self.modules.start() - self.addCleanup(self.modules.stop) - self.capacity = conversion["IMAGE_BATCH_SIZE"] - self.count = self.capacity * 2 + 1 - Model.rows = [ - {"id": identifier, "active": identifier != 3, "eligible": identifier != 7} for identifier in range(1, self.count + 1) - ] - - def test_native_selection_is_bounded_without_restricting_nested_searches(self) -> None: - seen = [] - nested = [] - - def native(model: Model) -> None: - selected = model.search([("eligible", "=", True)]).ids - model.env.loaded.update(selected) - if len(model.env.loaded) > model.env.capacity: - raise MemoryError("native image cache exceeded its capacity") - seen.extend(selected) - nested.append(model.search([("id", "=", self.count)]).ids) - - bounded = conversion["bounded_image_conversion"](native) - bounded(Model(Environment(self.capacity))) - expected = [row["id"] for row in Model.rows if row["active"] and row["eligible"]] - self.assertEqual(seen, expected) - self.assertTrue(all(ids == [self.count] for ids in nested)) - globals_ = conversion["bounded_image_conversion"].__globals__ - with patch.dict(globals_, {"scope_native_template_search": lambda *_: nullcontext()}): - with self.assertRaises(MemoryError): - bounded(Model(Environment(self.capacity))) - - def test_native_inactive_context_and_explicit_record_subset_survive(self) -> None: - seen = [] - bounded = conversion["bounded_image_conversion"](lambda model: seen.extend(model.search([("eligible", "=", True)]).ids)) - bounded(Model(Environment(self.capacity), active_test=False)) - self.assertEqual(seen, [row["id"] for row in Model.rows if row["eligible"]]) - seen.clear() - bounded(Model(Environment(self.capacity), [2, 3, 7], active_test=False)) - self.assertEqual(seen, [2, 3]) - - def test_native_failure_restores_local_and_inherited_search_methods(self) -> None: - class Inherited(Model): - pass - - def fail(_model: Model) -> None: - raise RuntimeError("native failure before its first search") - - for owner in (Model, Inherited): - original = owner.search - with self.assertRaises(RuntimeError): - conversion["bounded_image_conversion"](fail)(owner(Environment(self.capacity))) - self.assertIs(owner.search, original) - self.assertEqual(owner(Environment(self.capacity)).search([]).ids, [row["id"] for row in Model.rows if row["active"]]) From 6005cc213c832ba01d9d8d9b3c70f3c306d9bda4 Mon Sep 17 00:00:00 2001 From: shiny-code-bot <332303085+shiny-code-app[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 12:26:40 -0500 Subject: [PATCH 4/5] Compare batched scope with native model ordering --- .../addons/devkit_live_checks/tests/test_live_workflows.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py b/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py index cdf3a88..edf159c 100644 --- a/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py +++ b/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py @@ -82,20 +82,20 @@ def test_native_batch_scope_uses_real_recordsets_and_restores_search(self) -> No nested = [] def native(selected: Any) -> None: - observed.extend(selected.search([("id", "in", records.ids)], order="id").ids) + observed.extend(selected.search([("id", "in", records.ids)]).ids) nested.append(selected.search([("id", "=", records[-1].id)]).ids) original = type(model).search with patch.dict(conversion["bounded_image_conversion"].__globals__, {"IMAGE_BATCH_SIZE": 1}): conversion["bounded_image_conversion"](native)(model) - self.assertEqual(observed, records.ids) + self.assertEqual(observed, model.search([("id", "in", records.ids)]).ids) self.assertTrue(all(ids == [records[-1].id] for ids in nested)) self.assertIs(type(model).search, original) observed.clear() globals_ = conversion["bounded_image_conversion"].__globals__ with patch.dict(globals_, {"IMAGE_BATCH_SIZE": 1, "scope_native_template_search": lambda *_: nullcontext()}): conversion["bounded_image_conversion"](native)(model) - self.assertNotEqual(observed, records.ids) + self.assertNotEqual(observed, model.search([("id", "in", records.ids)]).ids) def fail(_records: Any) -> None: raise RuntimeError("native failure before the first search") From 346328bae8d791f1de714cdd23f7bd0757894c77 Mon Sep 17 00:00:00 2001 From: shiny-code-bot <332303085+shiny-code-app[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:20:42 -0500 Subject: [PATCH 5/5] Assert the native image search shape before batching --- docker/scripts/product_images_20.py | 1 + docs/tooling/workspace-cli.md | 2 ++ .../devkit_live_checks/tests/test_live_workflows.py | 9 ++++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/scripts/product_images_20.py b/docker/scripts/product_images_20.py index 75831c8..0f4c7bf 100644 --- a/docker/scripts/product_images_20.py +++ b/docker/scripts/product_images_20.py @@ -20,6 +20,7 @@ def scope_native_template_search(owner: Any, selected: list[int]) -> Iterator[No def search(records: Any, domain: Any, *args: Any, **kwargs: Any) -> Any: nonlocal first_search if first_search: + assert any(term.field_expr == "image_1920" for term in Domain(domain).iter_conditions()) first_search = False domain = Domain(domain) & Domain("id", "in", selected) return original_search(records, domain, *args, **kwargs) diff --git a/docs/tooling/workspace-cli.md b/docs/tooling/workspace-cli.md index 4f7c1cf..1be22e1 100644 --- a/docs/tooling/workspace-cli.md +++ b/docs/tooling/workspace-cli.md @@ -693,6 +693,8 @@ Notes keep their existing refusal behavior. The Website Sale pre-stage calls the native image conversion in bounded groups, retaining native selection and image associations without committing between groups. + Attachment content reads memoize file bytes in the ORM raw cache. Each group + flushes and invalidates that cache; the first scoped search must name `image_1920`. Original business attachments remain strict. The saved19 driver recognizes new extra-image copies only when database ownership, source checksums and actual bytes match their own template or variant; rollback leaves no separate copy journal. diff --git a/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py b/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py index edf159c..f05a192 100644 --- a/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py +++ b/tests/live_odoo/addons/devkit_live_checks/tests/test_live_workflows.py @@ -82,7 +82,7 @@ def test_native_batch_scope_uses_real_recordsets_and_restores_search(self) -> No nested = [] def native(selected: Any) -> None: - observed.extend(selected.search([("id", "in", records.ids)]).ids) + observed.extend(selected.search([("id", "in", records.ids), ("image_1920", "=", False)]).ids) nested.append(selected.search([("id", "=", records[-1].id)]).ids) original = type(model).search @@ -107,6 +107,13 @@ def fail(_records: Any) -> None: self.assertIs(type(model).search, original) self.assertEqual(model.search([("id", "in", records.ids)], order="id").ids, records.ids) + def changed_native(selected: Any) -> None: + selected.search([("id", "in", records.ids)]) + + with self.assertRaises(AssertionError): + conversion["bounded_image_conversion"](changed_native)(model) + self.assertIs(type(model).search, original) + def test_native_image_copy_ownership_and_shared_file_use_real_sql(self) -> None: check = runpy.run_path("/volumes/scripts/product_images_20.py")["native_image_copies"] with (