From e1b266b2ada688bc785d56d6d074cab1c088cd30 Mon Sep 17 00:00:00 2001 From: Jahongir Qurbonov Date: Sat, 14 Nov 2026 16:08:40 +0500 Subject: [PATCH 1/2] [mypyc] Regenerate dependent modules when weakrefs is toggled on a base class @mypyc_attr(weakrefs=True) (added in #22387) adds a weak reference list field to the object struct of the class, which moves the attributes of all native subclasses. With separate compilation, toggling it on a base class didn't regenerate subclasses in other groups, since mypy doesn't record class decorators and the interface hash of the base module stayed the same. The subclass kept its stale layout, and accessing its attributes or creating a weak reference could segfault. Include the weakrefs flag in class_kinds(), which MypycPlugin reports as part of a module's interface. When writing the cache it's derived from the tree like prepare_class_def sets ClassIR._weakrefs, and when checking the cache it's read from the cached IR. --- mypyc/codegen/emitmodule.py | 40 ++++++++------ mypyc/test-data/run-multimodule.test | 82 ++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 0f5093f3a258c..4e6feb5b757e8 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -79,7 +79,7 @@ from mypyc.irbuild.main import build_ir from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.prepare import load_type_map -from mypyc.irbuild.util import is_decorated_class, is_extension_class, is_trait +from mypyc.irbuild.util import get_mypyc_attrs, is_decorated_class, is_extension_class, is_trait from mypyc.namegen import NameGenerator, exported_name from mypyc.options import CompilerOptions from mypyc.transform.borrow_generator_attrs import borrow_generator_attrs @@ -141,8 +141,9 @@ class MypycPlugin(Plugin): other modules use a class: @mypyc_attr(native_class=False) and @trait decide what kind of class mypyc generates, and a decorator that can replace the class changes what its name refers to (see - ClassIR.is_decorated). So we make the kinds of a module's classes - part of its interface. + ClassIR.is_decorated). @mypyc_attr(weakrefs=True) changes the object + layout of subclasses. So we make the kinds of a module's classes part + of its interface. """ def __init__( @@ -160,7 +161,7 @@ def __init__( def report_config_data( self, ctx: ReportConfigContext - ) -> tuple[str | None, list[str], list[tuple[str, bool, bool, bool]]] | None: + ) -> tuple[str | None, list[str], list[tuple[str, bool, bool, bool, bool]]] | None: # The config data we report is the group map entry for the module, # followed by the kinds of its classes (see class_kinds). # If the data is being used to check validity, we do additional checks @@ -212,7 +213,7 @@ def report_config_data( # The module hasn't been parsed yet, so take the class kinds from the # cached IR. It was built from the source the cache is for. kinds = sorted( - (c["name"], c["is_ext_class"], c["is_trait"], c["is_decorated"]) + (c["name"], c["is_ext_class"], c["is_trait"], c["is_decorated"], c["_weakrefs"]) for c in ir_data["ir"]["classes"] if not c["is_generated"] ) @@ -245,23 +246,26 @@ def get_additional_indirect_deps(self, file: MypyFile) -> set[str]: return mods -def class_kinds(tree: MypyFile, options: Options) -> list[tuple[str, bool, bool, bool]]: - """Return (name, is_ext_class, is_trait, is_decorated) for each class in a module. +def class_kinds(tree: MypyFile, options: Options) -> list[tuple[str, bool, bool, bool, bool]]: + """Return (name, is_ext_class, is_trait, is_decorated, weakrefs) for each class in a module. - These match the ClassIR flags that build_type_map sets. + These match the ClassIR flags that build_type_map sets (weakrefs is + ClassIR._weakrefs, which prepare_class_def sets). """ # Any errors are reported when the module is compiled. errors = Errors(options) - return sorted( - ( - node.name, - is_extension_class(tree.path, node, errors), - is_trait(node), - is_decorated_class(node), - ) - for node in tree.defs - if isinstance(node, ClassDef) - ) + kinds = [] + for node in tree.defs: + if isinstance(node, ClassDef): + ext_class = is_extension_class(tree.path, node, errors) + trait = is_trait(node) + weakrefs = ( + ext_class + and not trait + and get_mypyc_attrs(node, tree.path, errors)[0].get("weakrefs") is True + ) + kinds.append((node.name, ext_class, trait, is_decorated_class(node), weakrefs)) + return sorted(kinds) def parse_and_typecheck( diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 3c8b4ceb99957..66278b3936a1e 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -1903,6 +1903,88 @@ print(call(D())) [rechecked2 native] [rechecked3 native, other] +[case testIncrementalCompilationTogglingWeakrefs] +# Mypy also ignores @mypyc_attr(weakrefs=True). It adds a weak reference list +# to the object struct of A, which moves the attributes of subclasses, so +# native.py must be recompiled to give B a layout that matches. +# In step 3 other.py is unchanged, so it must stay fresh. +from other import A + +class B(A): + def __init__(self) -> None: + super().__init__() + self.y = 2 + self.t = "b" + +[file other.py] +from mypy_extensions import mypyc_attr + +class A: + def __init__(self) -> None: + self.x = 1 + self.s = "a" + +def get_x(a: A) -> int: + return a.x + +[file other.py.2] +from mypy_extensions import mypyc_attr + +@mypyc_attr(weakrefs=True) +class A: + def __init__(self) -> None: + self.x = 1 + self.s = "a" + +def get_x(a: A) -> int: + return a.x + +[file native.py.3] +from other import A + +class B(A): + def __init__(self) -> None: + super().__init__() + self.y = 3 + self.t = "b" + +[file other.py.4] +from mypy_extensions import mypyc_attr + +class A: + def __init__(self) -> None: + self.x = 1 + self.s = "a" + +def get_x(a: A) -> int: + return a.x + +[file driver.py] +import weakref +from native import B +from other import get_x + +b = B() +try: + r = weakref.ref(b) +except TypeError: + weak = False +else: + weak = r() is b +print(get_x(b), b.x, b.s, b.y, b.t, weak) +[out] +1 1 a 2 b False +[out2] +1 1 a 2 b True +[out3] +1 1 a 3 b True +[out4] +1 1 a 3 b False + +[rechecked native, other] +[rechecked2 native] +[rechecked3 native, other] + [case testExtraLibRtSourceFileDep] table_list = list(range(256)) table_list[ord('A')] = ord('B') From 600da253d1aba53c10ac1b5026c479accfd044c7 Mon Sep 17 00:00:00 2001 From: Jahongir Qurbonov Date: Sat, 14 Nov 2026 17:07:23 +0500 Subject: [PATCH 2/2] [mypyc] Regenerate dependent modules when serializable or acyclic is toggled Like weakrefs, the serializable and acyclic flags of `@mypyc_attr` affect the generated code of subclasses in other modules, but mypy ignores the decorator, so toggling them didn't change the interface of the class's module. With separate compilation, the subclasses were then left stale: * `serializable` is inherited, and the tp_new of a serializable class doesn't call `__init__`. After making a base class serializable, a stale subclass still called `__init__` without arguments from tp_new, so `copy.copy()` and `pickle` failed with a `TypeError` about missing arguments to `__init__`. * A class can only be acyclic if all its base classes are. After making a base class non-acyclic, the error for an acyclic subclass wasn't reported. CPython then gives the subclass the GC flag and tp_traverse of the base, but the subclass's tp_dealloc doesn't untrack instances, so a collection triggered while an instance was being freed (e.g. from a `__del__` of one of its attributes) crashed. Add both flags to the class kinds that `MypycPlugin.report_config_data` makes part of a module's interface, using the `_serializable` and `is_acyclic` fields of the cached IR on the check path. --- mypyc/codegen/emitmodule.py | 55 ++++++++---- mypyc/test-data/run-multimodule.test | 121 +++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 14 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 4e6feb5b757e8..8cd5aff42e3b3 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -79,7 +79,13 @@ from mypyc.irbuild.main import build_ir from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.prepare import load_type_map -from mypyc.irbuild.util import get_mypyc_attrs, is_decorated_class, is_extension_class, is_trait +from mypyc.irbuild.util import ( + MypycAttrs, + get_mypyc_attrs, + is_decorated_class, + is_extension_class, + is_trait, +) from mypyc.namegen import NameGenerator, exported_name from mypyc.options import CompilerOptions from mypyc.transform.borrow_generator_attrs import borrow_generator_attrs @@ -118,6 +124,9 @@ # A list of (file name, file contents) pairs. FileContents = list[tuple[str, str]] +# The kind of a class that other modules depend on (see class_kinds). +ClassKind = tuple[str, bool, bool, bool, bool, bool, bool] + class MarkedDeclaration: """Add a mark, useful for topological sort.""" @@ -142,8 +151,10 @@ class MypycPlugin(Plugin): @trait decide what kind of class mypyc generates, and a decorator that can replace the class changes what its name refers to (see ClassIR.is_decorated). @mypyc_attr(weakrefs=True) changes the object - layout of subclasses. So we make the kinds of a module's classes part - of its interface. + layout of subclasses, @mypyc_attr(serializable=True) is inherited by + subclasses and changes how they are constructed, and + @mypyc_attr(acyclic=True) decides whether subclasses can be acyclic. + So we make the kinds of a module's classes part of its interface. """ def __init__( @@ -161,7 +172,7 @@ def __init__( def report_config_data( self, ctx: ReportConfigContext - ) -> tuple[str | None, list[str], list[tuple[str, bool, bool, bool, bool]]] | None: + ) -> tuple[str | None, list[str], list[ClassKind]] | None: # The config data we report is the group map entry for the module, # followed by the kinds of its classes (see class_kinds). # If the data is being used to check validity, we do additional checks @@ -213,7 +224,15 @@ def report_config_data( # The module hasn't been parsed yet, so take the class kinds from the # cached IR. It was built from the source the cache is for. kinds = sorted( - (c["name"], c["is_ext_class"], c["is_trait"], c["is_decorated"], c["_weakrefs"]) + ( + c["name"], + c["is_ext_class"], + c["is_trait"], + c["is_decorated"], + c["_weakrefs"], + c["_serializable"], + c.get("is_acyclic", False), + ) for c in ir_data["ir"]["classes"] if not c["is_generated"] ) @@ -246,11 +265,13 @@ def get_additional_indirect_deps(self, file: MypyFile) -> set[str]: return mods -def class_kinds(tree: MypyFile, options: Options) -> list[tuple[str, bool, bool, bool, bool]]: - """Return (name, is_ext_class, is_trait, is_decorated, weakrefs) for each class in a module. +def class_kinds(tree: MypyFile, options: Options) -> list[ClassKind]: + """Return the kind of each class in a module. - These match the ClassIR flags that build_type_map sets (weakrefs is - ClassIR._weakrefs, which prepare_class_def sets). + The kind of a class is (name, is_ext_class, is_trait, is_decorated, + weakrefs, serializable, acyclic). These match the ClassIR flags that + build_type_map sets, and the ClassIR flags _weakrefs, _serializable and + is_acyclic, which prepare_class_def sets for native classes. """ # Any errors are reported when the module is compiled. errors = Errors(options) @@ -259,12 +280,18 @@ def class_kinds(tree: MypyFile, options: Options) -> list[tuple[str, bool, bool, if isinstance(node, ClassDef): ext_class = is_extension_class(tree.path, node, errors) trait = is_trait(node) - weakrefs = ( - ext_class - and not trait - and get_mypyc_attrs(node, tree.path, errors)[0].get("weakrefs") is True + attrs: MypycAttrs = get_mypyc_attrs(node, tree.path, errors)[0] if ext_class else {} + kinds.append( + ( + node.name, + ext_class, + trait, + is_decorated_class(node), + not trait and attrs.get("weakrefs") is True, + attrs.get("serializable") is True, + attrs.get("acyclic") is True, + ) ) - kinds.append((node.name, ext_class, trait, is_decorated_class(node), weakrefs)) return sorted(kinds) diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 66278b3936a1e..c60f3a756503d 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -1985,6 +1985,127 @@ print(get_x(b), b.x, b.s, b.y, b.t, weak) [rechecked2 native] [rechecked3 native, other] +[case testIncrementalCompilationTogglingSerializable] +# Mypy also ignores @mypyc_attr(serializable=True). B inherits the flag from +# A, and then B.__new__ must not call __init__, since copy.copy() calls it +# without arguments. So native.py must be recompiled. +# In step 3 other.py is unchanged, so it must stay fresh. +from other import A + +class B(A): + def __init__(self, x: int, y: int) -> None: + super().__init__(x) + self.y = y + +[file other.py] +from mypy_extensions import mypyc_attr + +class A: + def __init__(self, x: int) -> None: + self.x = x + +[file other.py.2] +from mypy_extensions import mypyc_attr + +@mypyc_attr(serializable=True) +class A: + def __init__(self, x: int) -> None: + self.x = x + +[file native.py.3] +from other import A + +class B(A): + def __init__(self, x: int, y: int) -> None: + super().__init__(x) + self.y = y + 1 + +[file other.py.4] +from mypy_extensions import mypyc_attr + +class A: + def __init__(self, x: int) -> None: + self.x = x + +[file driver.py] +import copy +from native import B + +b = B(1, 2) +try: + c = copy.copy(b) +except TypeError: + print(b.x, b.y, "TypeError") +else: + print(c.x, c.y, c is not b) +[out] +1 2 TypeError +[out2] +1 2 True +[out3] +1 3 True +[out4] +1 3 TypeError + +[rechecked native, other] +[rechecked2 native] +[rechecked3 native, other] + +[case testIncrementalCompilationTogglingAcyclic] +# Mypy also ignores @mypyc_attr(acyclic=True). A class can only be acyclic if +# its base classes are, so native.py must be rechecked when A changes. +# Otherwise, making A non-acyclic again after step 3 wouldn't be reported as +# an error, and freeing a B could crash, since CPython would make B a GC type +# that the tp_dealloc of B doesn't untrack. +# In step 3 only native.py changes, so other.py must stay fresh. +from other import A + +class B(A): + def __init__(self) -> None: + super().__init__() + self.y: object = None + +[file other.py] +from mypy_extensions import mypyc_attr + +class A: + def __init__(self) -> None: + self.x: object = None + +[file other.py.2] +from mypy_extensions import mypyc_attr + +@mypyc_attr(acyclic=True) +class A: + def __init__(self) -> None: + self.x: object = None + +[file native.py.3] +from mypy_extensions import mypyc_attr +from other import A + +@mypyc_attr(acyclic=True) +class B(A): + def __init__(self) -> None: + super().__init__() + self.y: object = None + +[file driver.py] +import gc +from native import B +from other import A + +print(gc.is_tracked(A()), gc.is_tracked(B())) +[out] +True True +[out2] +False True +[out3] +False False + +[rechecked native, other] +[rechecked2 native] + [case testExtraLibRtSourceFileDep] table_list = list(range(256)) table_list[ord('A')] = ord('B')