diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 55f8e046e35e14b..7ed70505a21a4bc 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -2,6 +2,18 @@ # error "this header file must not be included directly" #endif +/* A region reference is a weak reference that survives its target's region + * being closed. Instead of keeping the region open it checks on every + * dereference whether this interpreter may reach the target, and opens the + * region tree if it may. The metadata carrying that information lives in + * `pycore_regionref.h`; it is opaque here. + */ +PyAPI_DATA(PyTypeObject) _PyRegionref_RefType; + +#define _PyRegionRef_CheckExact(op) Py_IS_TYPE((op), &_PyRegionref_RefType) + +struct _PyRegionRefMetadata; + /* PyWeakReference is the base struct for the Python ReferenceType, ProxyType, * and CallableProxyType. */ @@ -43,12 +55,23 @@ struct _PyWeakReference { */ PyMutex *weakrefs_lock; #endif + + /* The ownership domain of `wr_object`, or NULL if this object doesn't have an ownership + * domain. This can happen if this is a normal weakref or if the object is immutable. + */ + struct _PyRegionRefMetadata *region_ref; }; PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); +/* Region references reuse this struct but are deliberately not a subtype of + * `_PyWeakref_RefType`, so that `PyWeakref_Check()` stays false for them and + * the region close trace does not follow them. */ +#define _PyWeakrefOrRegionRef_Check(op) \ + (PyWeakref_Check(op) || _PyRegionRef_CheckExact(op)) + #define _PyWeakref_CAST(op) \ - (assert(PyWeakref_Check(op)), _Py_CAST(PyWeakReference*, (op))) + (assert(_PyWeakrefOrRegionRef_Check(op)), _Py_CAST(PyWeakReference*, (op))) // Test if a weak reference is dead. PyAPI_FUNC(int) PyWeakref_IsDead(PyObject *ref); diff --git a/Include/internal/pycore_cown.h b/Include/internal/pycore_cown.h new file mode 100644 index 000000000000000..60fe3058df61bd6 --- /dev/null +++ b/Include/internal/pycore_cown.h @@ -0,0 +1,39 @@ +#ifndef Py_INTERNAL_COWN_H +#define Py_INTERNAL_COWN_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +#include "object.h" +#include "exports.h" + +typedef struct _PyCownObject _PyCownObject; +#define _PyCownObject_CAST(op) _Py_CAST(_PyCownObject*, op) + +PyAPI_DATA(PyTypeObject) _PyCown_Type; + +typedef uint64_t _PyCown_ipid_t; +typedef uint64_t _PyCown_thread_id_t; + +PyAPI_FUNC(_PyCown_ipid_t) _PyCown_ThisInterpreterId(void); +PyAPI_FUNC(_PyCown_thread_id_t) _PyCown_ThisThreadId(void); + +/* The interpreter currently owning the cown, or `_PyCown_ReleasedIpid()` when + * no interpreter does. Safe to call from any interpreter. */ +PyAPI_FUNC(_PyCown_ipid_t) _PyCown_Owner(PyObject *cown); +PyAPI_FUNC(_PyCown_ipid_t) _PyCown_ReleasedIpid(void); + +/* The thread that acquired the cown, or `_PyCown_UnsetThreadId()` when it was + * acquired without the GIL. Not enforced, only reported. */ +PyAPI_FUNC(_PyCown_thread_id_t) _PyCown_LockingThread(PyObject *cown); +PyAPI_FUNC(_PyCown_thread_id_t) _PyCown_UnsetThreadId(void); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_COWN_H */ \ No newline at end of file diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 2dfce32237a83c3..6a1f91d2bad7cde 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -352,6 +352,25 @@ extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, int generation); extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists +/* Disposal of a list of objects that are known to be unreachable. Used by the + * collector itself and by anything else that owns a set of objects it has + * established to be garbage, such as a closed tracing region. + * + * `_PyGC_FinalizeGarbage()` runs the finalizer of every object in `collectable`, + * before anything is cleared, so that a `__del__` still sees its object intact. + * + * `_PyGC_DeleteGarbage()` then breaks the references between them, deallocating + * every object whose reference count reaches zero. Objects that a finalizer kept + * alive are moved to `old` instead. + * + * Neither may be called with an exception set. Only available in the default + * build; the free-threaded collector has its own implementation. + */ +#ifndef Py_GIL_DISABLED +extern void _PyGC_FinalizeGarbage(PyGC_Head *collectable); +extern void _PyGC_DeleteGarbage(PyGC_Head *collectable, PyGC_Head *old); +#endif + extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); diff --git a/Include/internal/pycore_immutability.h b/Include/internal/pycore_immutability.h index 8e4d32b78527a63..d883e44c69c7268 100644 --- a/Include/internal/pycore_immutability.h +++ b/Include/internal/pycore_immutability.h @@ -8,6 +8,33 @@ extern "C" { # error "Py_BUILD_CORE must be defined to include this header" #endif +struct _PyRegionRefMetadata; + +PyAPI_DATA(PyTypeObject) _PyTracingRegion_Type; +PyAPI_FUNC(int) _PyTracingRegion_Close(PyObject* region); +PyAPI_FUNC(int) _PyTracingRegion_IsClosed(PyObject* region); +PyAPI_FUNC(void) _PyTracingRegion_Open(PyObject* region); + +/* Returns the region's metadata node, allocating it if this is the first + * region reference the current close has found. Borrowed, and only valid while + * the region stays closed. The caller must hold `_PyWeakref_Lock`. */ +PyAPI_FUNC(struct _PyRegionRefMetadata*) + _PyTracingRegion_MetaLockHeld(PyObject* region); + +/* Hands a closed region's node to `cown`, used when a cown takes ownership of + * the region. Does nothing for an open region. */ +PyAPI_FUNC(void) _PyTracingRegion_SetMetaCown(PyObject* region, PyObject* cown); + +/* Records who owns a closed region, used when it leaves the cown that owned it. + * The owner is NOT necessarily the calling interpreter: a cown is immutable and + * may be deallocated by anyone holding a reference, including an interpreter + * that never owned it. Pass `_PyCown_ReleasedIpid()` when nobody owns it. + * Does nothing for an open region. */ +// FIXME: The deallocation will be fixed in a follow-up, then we can remove the +// owner argument and assert that it's always local. +PyAPI_FUNC(void) _PyTracingRegion_SetMetaOwner( + PyObject* region, uint64_t owner); + struct _Py_immutability_state { int late_init_done; struct _Py_hashtable_t *shallow_immutable_types; diff --git a/Include/internal/pycore_regionref.h b/Include/internal/pycore_regionref.h new file mode 100644 index 000000000000000..acdff81d6df39e9 --- /dev/null +++ b/Include/internal/pycore_regionref.h @@ -0,0 +1,86 @@ +#ifndef Py_INTERNAL_REGIONREF_H +#define Py_INTERNAL_REGIONREF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +#include "pycore_cown.h" // _PyCown_ipid_t +#include "pycore_hashtable.h" // _Py_hashtable_t + +/* Every `RegionRef` points at a `_PyRegionRefMetadata` node, and those nodes + * form a tree that mirrors the region hierarchy: a nested region's node + * delegates to the node of its parent, and the outermost node names the owner, + * either an interpreter or the cown holding the region. A cown node looks the + * owner up on the cown, which is what makes acquiring and releasing free; + * moving a closed region between owners restamps a single node. + * + * A reference to an object that is in no region at all owns its own node. + */ + +typedef enum { + /* A close is in progress. The referenced data is unavailable but should be + * soon; dereferencing fails and the caller may retry. Only ever reachable + * through a child's `parent`, never directly from a reference. */ + _Py_REGION_REF_WIP, + /* Delegates to `value.parent`, the node of the enclosing region. */ + _Py_REGION_REF_META, + /* Terminal. The region is held by `value.cown`, on which the owner is + * looked up dynamically. */ + _Py_REGION_REF_COWN, + /* Terminal, owned by one interpreter. */ + _Py_REGION_REF_IPID, +} _PyRegionRefKind; + +typedef struct _PyRegionRefMetadata { + uint32_t rc; + /* A `_PyRegionRefKind`. */ + uint8_t kind; + /* Borrowed. Set only on region nodes, and only while that region is + * closed. Names the region a dereference has to open on its way down. + * + * Borrowing is safe because the pointer is only followed after the terminal + * check established that this interpreter owns the region, and a region can + * only be deallocated by its owner. */ + PyObject *region; + union { + struct _PyRegionRefMetadata *parent; /* META */ + PyObject *cown; /* COWN, borrowed */ + _PyCown_ipid_t ipid; /* IPID */ + } value; +} _PyRegionRefMetadata; + +/* Creates the node of a region that is being closed. Returns a new reference. */ +extern _PyRegionRefMetadata *_PyRegionRef_NewRegionMetaLockHeld(PyObject *region); + +extern void _PyRegionRef_MetaDecref(_PyRegionRefMetadata *meta); + +// Ownership transitions. +extern void _PyRegionRef_MetaSetParentLockHeld(_PyRegionRefMetadata *meta, + _PyRegionRefMetadata *parent); +/* Hands the node to `cown`, which is borrowed. The owner is from then on + * whoever holds the cown. */ +extern void _PyRegionRef_MetaSetCown(_PyRegionRefMetadata *meta, PyObject *cown); +/* Stamps an explicit owner, which need not be the current interpreter and may + * be `_PyCown_ReleasedIpid()` to mean nobody owns the region. */ +extern void _PyRegionRef_MetaSetIpid(_PyRegionRefMetadata *meta, + _PyCown_ipid_t ipid); +extern void _PyRegionRef_MetaRegionOpened(_PyRegionRefMetadata *meta); +extern void _PyRegionRef_MetaResolveWip(_PyRegionRefMetadata *meta); + +/* Re-homes every `RegionRef` pointing at `obj` onto `region`'s node, allocating + * that node if this is the first reference the close has found. Every other + * weak reference to `obj` is cleared unless it is listed in `keep`. + * + * This is the region close hook; it replaces `_PyWeakref_ClearWeakRefsExcept()` + * for objects that are being closed into a region. */ +extern void _PyRegionRef_CloseWeakRefs(PyObject *obj, _Py_hashtable_t *keep, + PyObject *region); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_REGIONREF_H */ diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index 3f0e1b995ee43f5..cced88c3fdcce33 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -13,6 +13,15 @@ extern "C" { #include "pycore_object.h" // _Py_REF_IS_MERGED() #include "pycore_pyatomic_ft_wrappers.h" +/* Guards weakrefs to immutable objects, and all region reference metadata. + * Declared for both builds because `_PyRegionRefMetadata` uses it either way, + * while the weakref lists themselves are striped in free-threaded builds. */ +extern PyMutex _PyWeakref_Lock; + +#define LOCK_REGION_REF_META() \ + PyMutex_LockFlags(&_PyWeakref_Lock, _Py_LOCK_DONT_DETACH) +#define UNLOCK_REGION_REF_META() PyMutex_Unlock(&_PyWeakref_Lock) + #ifdef Py_GIL_DISABLED #define WEAKREF_LIST_LOCK(obj) \ @@ -37,9 +46,6 @@ extern "C" { #else -// Lock used for weakrefs to immutable objects -extern PyMutex _PyWeakref_Lock; - #define LOCK_WEAKREFS(obj) PyMutex_LockFlags(&_PyWeakref_Lock, _Py_LOCK_DONT_DETACH) #define UNLOCK_WEAKREFS(obj) PyMutex_Unlock(&_PyWeakref_Lock) @@ -108,7 +114,7 @@ static inline PyObject* get_ref_lock_held(PyWeakReference *ref, PyObject *obj) static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) { - assert(PyWeakref_Check(ref_obj)); + assert(_PyWeakrefOrRegionRef_Check(ref_obj)); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); PyObject *obj = _Py_atomic_load_ptr(&ref->wr_object); @@ -125,7 +131,7 @@ static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) { - assert(PyWeakref_Check(ref_obj)); + assert(_PyWeakrefOrRegionRef_Check(ref_obj)); int ret = 0; PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); @@ -149,6 +155,10 @@ extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyObject *obj); // intact. extern void _PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj); +// Same, but every weak reference listed in `keep` survives. The table is keyed +// by the weak reference objects themselves, not by their referents. +extern void _PyWeakref_ClearWeakRefsExcept(PyObject *obj, _Py_hashtable_t *keep); + PyAPI_FUNC(void) _PyWeakref_OnObjectFreeze(PyObject *object); PyAPI_FUNC(void) _PyImmutability_ClearWeakRefsWithCallback(PyObject *object, PyWeakReference **callbacks); PyAPI_FUNC(int) _PyWeakref_IsDead(PyObject *weakref); diff --git a/Lib/immutable.py b/Lib/immutable.py index e1c00152f94bbd8..30ec5b32b2532d0 100644 --- a/Lib/immutable.py +++ b/Lib/immutable.py @@ -21,6 +21,9 @@ FREEZABLE_PROXY = _c.FREEZABLE_PROXY InterpreterLocal = _c.InterpreterLocal SharedField = _c.SharedField +TracingRegion = _c.TracingRegion +Cown = _c.Cown +RegionRef = _c.RegionRef # FIXME(immutable): For the longest time we used the name `isfrozen` # without the underscore. This keeps the function name for now, but @@ -139,6 +142,9 @@ def __enter__(self): "FREEZABLE_PROXY", "InterpreterLocal", "SharedField", + "TracingRegion", + "Cown", + "RegionRef", "freezable", "unfreezable", "explicitlyFreezable", diff --git a/Lib/test/test_freeze/test_implicit.py b/Lib/test/test_freeze/test_implicit.py index b710b787fe5bdaf..35e46036e66d75f 100644 --- a/Lib/test/test_freeze/test_implicit.py +++ b/Lib/test/test_freeze/test_implicit.py @@ -1,3 +1,4 @@ +import sys import unittest from immutable import freeze, is_frozen @@ -139,6 +140,26 @@ def test_deeply_nested_no_stack_overflow(self): obj = (obj,) self.assertTrue(is_frozen(obj)) + def test_abandoned_walk_keeps_references(self): + """An aborted walk must not drop references it never took. + + The walk pushes objects onto a worklist without increfing them, so + anything still on the worklist when a mutable object aborts the walk + used to be decrefed when the worklist was released. That freed the + object while its real owners were still pointing at it, which showed + up much later as a negative refcount. + """ + # Built at runtime so it is neither interned nor immortal, which makes + # its reference count fully accounted for by this test. + item = "".join(["abandoned", "-", "worklist", "-", "entry"]) + # Tuples are traversed back to front, so `item` reaches the worklist + # before the dict aborts the walk. + obj = ({"mutable": 1}, item) + + before = sys.getrefcount(item) + self.assertFalse(is_frozen(obj)) + self.assertEqual(sys.getrefcount(item), before) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_freeze/test_tracing_region.py b/Lib/test/test_freeze/test_tracing_region.py new file mode 100644 index 000000000000000..ae1701519cf921c --- /dev/null +++ b/Lib/test/test_freeze/test_tracing_region.py @@ -0,0 +1,987 @@ +import gc +import re +import sys +import unittest +import weakref +from immutable import freeze, is_frozen, freezable +from immutable import TracingRegion as Region +from immutable import Cown, InterpreterLocal, RegionRef +from test.support import import_helper, os_helper + +def sort_region_error(msg): + """Normalize a 'region could not be closed' message by masking the object + addresses and sorting its per-object lines. Useful for deterministic test + assertions, since the addresses differ per run and the object order comes + from hashtable iteration and isn't stable.""" + header, *lines = re.sub(r"0x[0-9a-fA-F]+", "0x...", msg).splitlines() + return [header, *sorted(lines)] + +class TestTracing(unittest.TestCase): + def test_release_error(self): + x = [1] + y = [2] + + c = Cown(Region()) + c.value.x = x + c.value.y = y + + with self.assertRaises(RuntimeError) as cm: + c.release() + + self.assertEqual( + sort_region_error(str(cm.exception)), + [ + "The region could not be closed due to:", + "- 1 incoming reference to list '[1]'", + "- 1 incoming reference to list '[2]'" + ]) + + def test_release_error_capped_output(self): + # The object order in the error message is based on the address + # and therefore fairly random. All elements look the same of + # make testing stable. + l = [[1], [1], [1], [1], [1], [1], [1], [1]] + + c = Cown(Region()) + c.value.x = [] + + for i in range(len(l)): + c.value.x.append(l[i]) + + with self.assertRaises(RuntimeError) as cm: + c.release() + + self.assertEqual( + sort_region_error(str(cm.exception)), + [ + "The region could not be closed due to:", + "- 1 incoming reference to list '[1]'", + "- 1 incoming reference to list '[1]'", + "- 1 incoming reference to list '[1]'", + "- 1 incoming reference to list '[1]'", + "- 1 incoming reference to list '[1]'", + "- 3 references to other objects", + ]) + + # The cown should now be released + l = None + c.release() + + def test_release_error_in_subregion(self): + x = [1] + + c = Cown(Region()) + child = Region() + child.x = x + c.value.child = child + + with self.assertRaises(RuntimeError) as cm: + c.release() + + self.assertEqual( + sort_region_error(str(cm.exception)), + [ + "The region could not be closed due to:", + "- 1 incoming reference to list '[1]'", + ]) + + def test_failed_multi_parent_region_close(self): + r1 = Region() + r2 = Region() + r3 = Region() + r2.sub = r3 + r1.lst = [r3, r2, r3] + del r2 + del r3 + + c = Cown(r1) + del r1 + + with self.assertRaises(RuntimeError) as cm: + c.release() + + self.assertEqual( + sort_region_error(str(cm.exception)), + [ + "The region could not be closed due to:", + "- 3 incoming references to TracingRegion ''", + ]) + + + def test_failed_cyclic_region_close(self): + r1 = Region() + r2 = Region() + r3 = Region() + + r1.r2 = r2 + r2.r3 = r3 + r3.r1 = r1 + c = Cown(r1) + + del r1 + del r2 + del r3 + + with self.assertRaises(RuntimeError) as cm: + c.release() + + self.assertEqual( + sort_region_error(str(cm.exception)), + [ + "the region 0x... can not be closed as it attempts to reference one of its parent regions 0x...", + ]) + + def test_weak_ref_in_region(self): + @freezable + class A: + pass + + r1 = Region() + r1.obj = A() + r1.wref1 = weakref.ref(r1.obj) + wref2 = weakref.ref(r1.obj) + + c = Cown(r1) + del r1 + + # Releasing should clear all external weak references + c.release() + self.assertIsNone(wref2()); + + # All internal weak references should remain valid + c.acquire() + self.assertEqual(c.value.wref1(), c.value.obj); + + def test_weak_ref_to_bridge(self): + """ + The closing code and cowns currently assume that bridges can't have weak references. + This tests asserts this. We can add support for weak refs, but that would require some + engineering and the question is if this is even needed. + """ + + r1 = Region() + with self.assertRaises(TypeError) as err: + weakref.ref(r1) + + self.assertEqual(str(err.exception), "cannot create weak reference to 'TracingRegion' object") + + +class TestRegionOpening(unittest.TestCase): + def test_open_after_acquire(self): + c = Cown(Region()) + c.value.x = [] + self.assertFalse(c._is_closed()) + + c.release() + c.acquire() + + self.assertTrue(c._is_closed()) + c.value.x = None + self.assertFalse(c._is_closed()) + + def test_release_closed_region(self): + c = Cown(Region()) + c.value.x = [] + self.assertFalse(c._is_closed()) + + c.release() + c.acquire() + + self.assertTrue(c._is_closed()) + + c.release() + + def test_bridge_refs_keep_region_closed(self): + c = Cown(Region()) + c.release() + c.acquire() + self.assertTrue(c._is_closed()) + + # Adding new references to the bridge object should keep it closed. + # only attribute accesses should open it. + r1 = c.value + r2 = c.value + self.assertTrue(c._is_closed()) + + # However, these references should prevent the cown from being released + with self.assertRaises(RuntimeError) as cm: + c.release() + + self.assertEqual( + str(cm.exception), + "the cown couldn't be released, due to the bridge having incoming references") + + # The release should succeed once all refs have been killed + del r1 + del r2 + c.release() + + def test_sub_region_closing(self): + @freezable + class A: + pass + c = Cown(Region()) + c.value.a = A() + c.value.a.child = Region() + c.value.a.child.b = A() + + c.release() + c.acquire() + + r2 = c.value.a.child + c2 = Cown(r2) + + self.assertTrue(c2._is_closed()) + + def test_sub_region_multiple_refs(self): + @freezable + class A: + pass + c = Cown(Region()) + c.value.a = A() + sub = Region() + c.value.a.child_a = sub + c.value.a.child_b = sub + # A reference to the bridge of a sub-region counts as an incoming + # reference into the parent region, see + # test_ref_to_sub_region_bridge_keeps_parent_open. + del sub + + c.release() + c.acquire() + + r2 = c.value.a.child_a + c2 = Cown(r2) + + self.assertTrue(c2._is_closed()) + + def test_ref_to_sub_region_bridge_keeps_parent_open(self): + c1 = Cown(Region()) + c2 = Cown(Region()) + c1.value.child = c2.value + + self.assertFalse(c2._is_closed()) + + with self.assertRaises(RuntimeError) as cm: + c1.release() + + # Attempting to close the region c1 should have closed c2 and then + # failed due to the incoming reference to the bridge stored in c2 + self.assertTrue(c2._is_closed()) + + + self.assertEqual( + sort_region_error(str(cm.exception)), + [ + "The region could not be closed due to:", + "- 1 incoming reference to TracingRegion ''", + ]) + + + +class TestImplicitFreeze(unittest.TestCase): + def test_implicit_freeze_func(self): + @freezable + def some_func(): + pass + c = Cown(Region()) + + c.value.obj = some_func + self.assertFalse(is_frozen(c.value.obj)) + c.release() + c.acquire() + self.assertTrue(is_frozen(c.value.obj)) + + def test_implicit_freeze_type(self): + @freezable + class A: + pass + c = Cown(Region()) + + c.value.obj = A + self.assertFalse(is_frozen(c.value.obj)) + c.release() + c.acquire() + self.assertTrue(is_frozen(c.value.obj)) + + def test_implicit_freeze_module(self): + import random; + c = Cown(Region()) + + c.value.obj = random + self.assertFalse(is_frozen(c.value.obj)) + c.release() + c.acquire() + self.assertTrue(is_frozen(c.value.obj)) + + # Unimport module + sys.modules.pop("random", None) + sys.mut_modules.pop("random", None) + + def test_implicit_freeze_str(self): + c = Cown(Region()) + + c.value.obj = "Ducks are cool" + c.release() + c.acquire() + self.assertTrue(is_frozen(c.value.obj)) + + def test_implicit_freeze_int(self): + c = Cown(Region()) + + c.value.obj = 17 + c.release() + c.acquire() + self.assertTrue(is_frozen(c.value.obj)) + + +class TestClosedRegionTeardown(unittest.TestCase): + """A closed region disposes of its own contents. + + Closing proves nothing outside the region references its members, so the + death of the bridge object makes all of them garbage. The region finalizes + and clears them itself rather than handing them to the GC. + """ + + def test_cycles_reclaimed_without_the_collector(self): + """Check that cycles in closed regions are reclaimed without the collector""" + + @freezable + class Node: + pass + + def _live_nodes(): + """The number of nodes the collector can see.""" + return sum(1 for o in gc.get_objects() if type(o) is Node) + + gc.disable() + try: + before = _live_nodes() + + # Create a cycle + a = Node() + b = Node() + a.b = b + b.a = a + + # Move the cycle into a cown + c = Cown(Region()) + c.value.cycle = a + del a + del b + mid = _live_nodes() + + # Close the region + c.release() + del c + + leaked = _live_nodes() - before + finally: + gc.enable() + + self.assertEqual(mid, 2, "the cycle wasn't detected while the region is open") + self.assertEqual(leaked, 0, "closed region contents were not reclaimed") + + def test_finalizers_run(self): + local = InterpreterLocal(0) + + @freezable + class Recorder: + def __del__(self, local=local): + local.set(local.get() + 1) + + c = Cown(Region()) + for i in range(5): + setattr(c.value, "r%d" % i, Recorder()) + c.release() + + self.assertEqual(local.get(), 0) + del c + self.assertEqual(local.get(), 5) + + + def test_finalizer_can_modify_the_bridge(self): + local = InterpreterLocal(False) + + @freezable + class Reenter: + def __del__(self, local=local): + # This will open the region and also prove that the finalizer ran + local.set(self.bridge.reenter == self) + self.bridge.__dict__ = {} + + # Create a cycle, and allow Reenter to modify the bridge + c = Cown(Region()) + c.value.reenter = Reenter() + c.value.reenter.bridge = c.value + c.release() + del c + + self.assertTrue(local.get(), "the finalizer did not run or got the wrong object") + + def test_finalizer_revivial(self): + local_bridge = InterpreterLocal(None) + local_medic = InterpreterLocal(None) + + @freezable + class Medic: + def __del__(self, loca_bridge=local_bridge, local_medic=local_medic): + local_bridge.set(self.bridge) + local_medic.set(self) + + c1 = Cown(Region()) + c1.value.reviver = Medic() + c1.value.reviver.bridge = c1.value + c1.release() + del c1 + + # Retrieve the revived bridge + self.assertIsInstance(local_bridge.get(), Region); + c2 = Cown(local_bridge.get()) + local_bridge.set(None) + + with self.assertRaises(RuntimeError) as e: + c2.release() + self.assertTrue(str(e.exception).endswith("has been finalized and cannot be closed again")) + + # Check that the revived medic object is valid + self.assertIn("Medic object at 0x", str(local_medic.get())) + + +class TestRegionRef(unittest.TestCase): + """A region reference does not keep a region open. It checks on every + dereference whether this interpreter may reach the target, and opens the + region tree on the way.""" + + def _obj(self, tag=0): + @freezable + class A: + pass + obj = A() + obj.tag = tag + return obj + + def test_deref_while_open(self): + r = Region() + r.obj = self._obj(1) + rr = RegionRef(r.obj) + self.assertIs(rr(), r.obj) + + def test_survives_close_unlike_weakref(self): + """A close clears the plain weak references pointing into the region + but re-homes the region references instead.""" + r = Region() + r.obj = self._obj(2) + wref = weakref.ref(r.obj) + rr = RegionRef(r.obj) + + c = Cown(r) + del r + c.release() + + self.assertIsNone(wref()) + c.acquire() + self.assertEqual(rr().tag, 2) + + def test_denied_while_released(self): + r = Region() + r.obj = self._obj(3) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + + with self.assertRaises(RuntimeError) as cm: + rr() + self.assertIn("released cown", str(cm.exception)) + + def test_deref_opens_the_region(self): + r = Region() + r.obj = self._obj(4) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + c.acquire() + + self.assertTrue(c._is_closed()) + obj = rr() + self.assertFalse(c._is_closed()) + self.assertIs(obj, c.value.obj) + + def test_deref_opens_the_whole_chain(self): + """A reference into a nested region has to open every region above it, + not just the one holding the target.""" + child = Region() + child.obj = self._obj(5) + rr = RegionRef(child.obj) + r = Region() + r.child = child + del child + + c = Cown(r) + del r + c.release() + c.acquire() + + self.assertTrue(c._is_closed()) + self.assertEqual(rr().tag, 5) + self.assertFalse(c._is_closed()) + self.assertEqual(c.value.child.obj.tag, 5) + + def test_release_after_acquire_without_opening(self): + """A release does not re-trace an already closed region, so nothing + re-stamps its node. The cown node is what keeps the next owner able to + dereference.""" + r = Region() + r.obj = self._obj(6) + rr = RegionRef(r.obj) + c = Cown(r) + del r + + c.release() + c.acquire() + c.release() + + with self.assertRaises(RuntimeError): + rr() + + c.acquire() + self.assertEqual(rr().tag, 6) + + def test_region_outliving_its_cown(self): + r = Region() + r.obj = self._obj(7) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + c.acquire() + + escaped = c.value + del c + gc.collect() + + self.assertEqual(rr().tag, 7) + self.assertIsNotNone(escaped) + + def test_target_moving_between_regions(self): + """A close re-homes every reference to the objects it traced, so an + object that changed regions resolves through the one it lives in.""" + c1 = Cown(Region()) + c1.value.obj = self._obj(8) + rr = RegionRef(c1.value.obj) + c1.release() + c1.acquire() + + c2 = Cown(Region()) + c2.value.obj = c1.value.obj + c1.value.obj = None + + # Closing the old region should not restrict the region reference + c1.release() + self.assertEqual(rr().tag, 8) + + # Closing the owning reference should restrict the region reference + c2.release() + with self.assertRaises(RuntimeError): + rr() + + # Opening the owning cown allows the region reference again + c2.acquire() + self.assertEqual(rr().tag, 8) + + def test_dead_target(self): + r = Region() + r.obj = self._obj(9) + rr = RegionRef(r.obj) + r.obj = None + gc.collect() + + self.assertIsNone(rr()) + self.assertIn("dead", repr(rr)) + + def test_frozen_target_drops_the_check(self): + """A frozen object is reachable from everywhere, so its references stop + carrying an ownership check.""" + obj = self._obj(10) + rr = RegionRef(obj) + freeze(obj) + self.assertEqual(rr().tag, 10) + + def test_repr_does_not_open_the_region(self): + r = Region() + r.obj = self._obj(11) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + + self.assertIn("unavailable", repr(rr)) + + c.acquire() + self.assertTrue(c._is_closed()) + self.assertIn("to '", repr(rr)) + self.assertTrue(c._is_closed()) + + def test_no_callback_argument(self): + # FIXME(regions): Callbacks are not supported yet. + obj = self._obj(12) + with self.assertRaises(TypeError): + RegionRef(obj, lambda ref: None) + + def test_equality(self): + obj = self._obj(13) + other = self._obj(13) + self.assertEqual(RegionRef(obj), RegionRef(obj)) + self.assertNotEqual(RegionRef(obj), RegionRef(other)) + + def test_failed_close_leaves_the_reference_local(self): + """A close that fails leaves its node unresolved. It has to end up + local to this interpreter, or the reference would be stuck.""" + child = Region() + child.obj = self._obj(15) + rr = RegionRef(child.obj) + r = Region() + r.child = child + del child + + leak = self._obj(16) + r.leak = leak + c = Cown(r) + del r + + with self.assertRaises(RuntimeError): + c.release() + self.assertEqual(rr().tag, 15) + + def test_closed_but_not_released_leaves_the_reference_local(self): + """The tree can close and the cown still refuse to release. Nothing + roots the region in that case, so it stays ours.""" + r = Region() + r.obj = self._obj(17) + rr = RegionRef(r.obj) + c = Cown(r) + del r + + held = c.value + with self.assertRaises(RuntimeError) as cm: + c.release() + self.assertIn("incoming references", str(cm.exception)) + self.assertEqual(rr().tag, 17) + + del held + gc.collect() + c.release() + with self.assertRaises(RuntimeError): + rr() + + def test_deeply_nested_region(self): + inner = Region() + inner.obj = self._obj(18) + rr = RegionRef(inner.obj) + node = inner + for _ in range(30): + outer = Region() + outer.child = node + node = outer + c = Cown(node) + del node, inner, outer + + c.release() + c.acquire() + self.assertEqual(rr().tag, 18) + + def test_repeated_close_open_cycles(self): + """Each close allocates a fresh node and re-homes the reference onto + it. The old ones have to go away with it.""" + r = Region() + r.obj = self._obj(19) + rr = RegionRef(r.obj) + c = Cown(r) + del r + + for _ in range(100): + c.release() + c.acquire() + self.assertEqual(rr().tag, 19) + + def test_target_collected_as_cyclic_garbage(self): + """The collector clears weak references itself, without going through + the type. A RegionRef is not a weakref subtype, so every one of those + paths has to know about it.""" + obj = self._obj(20) + obj.self = obj + rr = RegionRef(obj) + del obj + gc.collect() + self.assertIsNone(rr()) + + def test_reference_itself_is_cyclic_garbage(self): + """A RegionRef that is itself unreachable has to be cleared before the + garbage is deleted, or a __del__ could still dereference it.""" + obj = self._obj(21) + cell = [RegionRef(obj)] + cell.append(cell) + del cell + gc.collect() + self.assertIsNotNone(obj) + + def test_not_freezable(self): + """Freezing would have to either drag the target into the frozen set or + let an immutable object reach a mutable one. Neither is acceptable, so + a reference is simply not freezable -- while its type still is, since + closing a region freezes the type of everything it moves.""" + obj = self._obj(22) + rr = RegionRef(obj) + with self.assertRaises(TypeError): + freeze(rr) + # The type itself stays freezable: closing a region freezes the type of + # everything it moves, so a reference could not live in one otherwise. + freeze(RegionRef) + self.assertTrue(is_frozen(RegionRef)) + # The target must still be able to die cleanly afterwards. + del obj + gc.collect() + self.assertIsNone(rr()) + + def test_already_frozen_target_is_unrestricted(self): + """Freezing before or after the reference is created has to give the + same answer, otherwise the semantics depend on ordering.""" + early = self._obj(23) + freeze(early) + rr_early = RegionRef(early) + + late = self._obj(24) + rr_late = RegionRef(late) + freeze(late) + + self.assertEqual(rr_early().tag, 23) + self.assertEqual(rr_late().tag, 24) + + def test_hash_is_checked_even_when_cached(self): + """A cached hash would otherwise stand as an answer about an object + this interpreter may no longer touch.""" + r = Region() + r.obj = self._obj(26) + rr = RegionRef(r.obj) + hash(rr) + + c = Cown(r) + del r + c.release() + with self.assertRaises(RuntimeError): + hash(rr) + + c.acquire() + self.assertIsInstance(hash(rr), int) + + def test_repr_preserves_a_pending_exception(self): + """repr() runs from error reporting paths, so a denied check must not + wipe the error state the caller is carrying.""" + r = Region() + r.obj = self._obj(25) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + + try: + raise ValueError("caller's error") + except ValueError: + self.assertIn("unavailable", repr(rr)) + self.assertIsInstance(sys.exception(), ValueError) + + def test_reference_inside_a_region(self): + """A region reference is not followed by the close trace, so storing + one in a region does not drag its target in.""" + outside = self._obj(14) + r = Region() + r.rr = RegionRef(outside) + c = Cown(r) + del r + + c.release() + self.assertFalse(is_frozen(outside)) + c.acquire() + self.assertIs(c.value.rr(), outside) + + +class TestRegionRefSubinterpreters(unittest.TestCase): + """The point of the ownership check: another interpreter may only + dereference what it actually owns.""" + + def setUp(self): + self._interpreters = import_helper.import_module('_interpreters') + + def _run_in_subinterp(self, code, shared=None): + interp = self._interpreters.create() + try: + self._interpreters.run_string(interp, code, shared=shared or {}) + finally: + self._interpreters.destroy(interp) + + def test_frozen_target_reachable_from_everywhere(self): + """A frozen target is shareable, so its references carry no ownership + check at all -- whichever order the freeze and the reference happened + in.""" + @freezable + class A: + pass + + early = A(); early.tag = "early" + freeze(early) + late = A(); late.tag = "late" + + r = Region() + r.early_ref = RegionRef(early) + r.late_ref = RegionRef(late) + freeze(late) + c = Cown(r) + del r + c.release() + + self._run_in_subinterp(""" +c.acquire() +assert c.value.early_ref().tag == "early", "frozen before the ref was created" +assert c.value.late_ref().tag == "late", "frozen after the ref was created" +c.release() +""", shared={"c": c}) + + def test_foreign_deallocation_does_not_transfer_ownership(self): + """A cown is immutable and may live inside a region, so the last + reference to it can be dropped by an interpreter that never owned it. + Its region must go to the cown's owner, not to whoever runs the + deallocator.""" + @freezable + class A: + pass + + inner_region = Region() + inner_region.obj = A() + inner_region.obj.tag = "owned by the creator" + mine = RegionRef(inner_region.obj) + travelling = RegionRef(inner_region.obj) + + inner = Cown(inner_region) + del inner_region + inner.release() + inner.acquire() + # Keep the region alive past the cown, so the cown can die alone. + escaped = inner.value + + outer_region = Region() + outer_region.inner = inner + outer_region.ref = travelling + del travelling + outer = Cown(outer_region) + del outer_region + outer.release() + del inner + gc.collect() + + self._run_in_subinterp(""" +import gc +c.acquire() +c.value.inner = None # drops the last reference to a cown we never owned +gc.collect() +try: + c.value.ref() + raise AssertionError("reached a region owned by another interpreter") +except RuntimeError: + pass +c.release() +""", shared={"c": outer}) + + # The creator still owns it, and is not locked out of its own region. + self.assertEqual(escaped.obj.tag, "owned by the creator") + self.assertEqual(mine().tag, "owned by the creator") + + def test_foreign_deallocation_defers_the_teardown_to_the_owner(self): + """The last reference to a cown owned by this interpreter may be + dropped by another one. The region inside is reference counted + non-atomically and tracked in this interpreter's GC list, so the + teardown has to be handed back here instead of running there.""" + log = os_helper.TESTFN + self.addCleanup(os_helper.unlink, log) + + @freezable + class Marker: + def __del__(self): + with open(self.log, "a") as f: + f.write("region\n") + + inner_region = Region() + inner_region.marker = Marker() + inner_region.marker.log = log + ref = RegionRef(inner_region.marker) + + inner = Cown(inner_region) + del inner_region + inner.release() + inner.acquire() # owned by this interpreter from here on + + outer_region = Region() + outer_region.inner = inner + outer = Cown(outer_region) + del outer_region + outer.release() + del inner + gc.collect() + + with open(log, "w"): + pass + + self._run_in_subinterp(""" +import gc +c.acquire() +c.value.inner = None # drops the last reference to a cown we never owned +gc.collect() +with open(log, "a") as f: + f.write("subinterpreter ") +c.release() +""", shared={"c": outer, "log": log}) + + # The teardown was scheduled here and runs at the next eval breaker. + recorded = [] + for _ in range(10000): + with open(log) as f: + recorded = f.read().split() + if len(recorded) == 2: + break + self.assertEqual(recorded, ["subinterpreter", "region"]) + self.assertIsNone(ref()) + + def test_owner_may_deref_and_others_may_not(self): + """`local` was never in a region, so nothing ever re-homes the + reference to it and it stays local to this interpreter. `owned` travels + with the region and becomes reachable by whoever holds the cown.""" + @freezable + class A: + pass + + local = A() + local.tag = "local" + + r = Region() + r.local_ref = RegionRef(local) + r.owned = A() + r.owned.tag = "owned" + r.owned_ref = RegionRef(r.owned) + c = Cown(r) + del r + c.release() + + self._run_in_subinterp(""" +c.acquire() +try: + c.value.local_ref() + raise AssertionError("reached a foreign local object") +except RuntimeError: + pass +assert c.value.owned_ref().tag == "owned" +c.release() +""", shared={"c": c}) + + c.acquire() + self.assertEqual(c.value.local_ref().tag, "local") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 424044e2466f149..9a00cee20505ff7 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1842,10 +1842,11 @@ class newstyleclass(object): pass # TODO: add check that forces layout of unicodefields # weakref import weakref + # The trailing pointer is `region_ref`, see `Include/cpython/weakrefobject.h`. if support.Py_GIL_DISABLED: - expected = size('2Pln4P') + expected = size('2Pln5P') else: - expected = size('2Pln3P') + expected = size('2Pln4P') check(weakref.ref(int), expected) # weakproxy # XXX diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 47f6b46061ac304..7309eecc0a8a0bc 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -266,6 +266,7 @@ def check_basic_callback(self, factory): self.assertIsNone(ref(), "ref2 should be dead after deleting object reference") + @unittest.skip("FIXME: Optimization is disabled to support weakrefs in regions") def test_ref_reuse(self): o = C() ref1 = weakref.ref(o) @@ -289,6 +290,7 @@ def test_ref_reuse(self): self.assertEqual(weakref.getweakrefcount(o), 1, "wrong weak ref count for object after deleting proxy") + @unittest.skip("FIXME: Optimization is disabled to support weakrefs in regions") def test_proxy_reuse(self): o = C() proxy1 = weakref.proxy(o) @@ -379,9 +381,11 @@ def __imatmul__(self, other): # was not honored, and was broken in different ways for # PyWeakref_NewRef() and PyWeakref_NewProxy(). (Two tests.) + @unittest.skip("FIXME: Optimization is disabled to support weakrefs in regions") def test_shared_ref_without_callback(self): self.check_shared_without_callback(weakref.ref) + @unittest.skip("FIXME: Optimization is disabled to support weakrefs in regions") def test_shared_proxy_without_callback(self): self.check_shared_without_callback(weakref.proxy) diff --git a/Makefile.pre.in b/Makefile.pre.in index 572a784546b60fb..15a7f4958719206 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -528,6 +528,7 @@ OBJECT_OBJS= \ Objects/classobject.o \ Objects/codeobject.o \ Objects/complexobject.o \ + Objects/cownobject.o \ Objects/descrobject.o \ Objects/enumobject.o \ Objects/exceptions.o \ @@ -555,6 +556,7 @@ OBJECT_OBJS= \ Objects/sliceobject.o \ Objects/structseq.o \ Objects/templateobject.o \ + Objects/tracingregionobject.o \ Objects/tupleobject.o \ Objects/typeobject.o \ Objects/typevarobject.o \ @@ -1333,6 +1335,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_complexobject.h \ $(srcdir)/Include/internal/pycore_condvar.h \ $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_cown.h \ $(srcdir)/Include/internal/pycore_critical_section.h \ $(srcdir)/Include/internal/pycore_crossinterp.h \ $(srcdir)/Include/internal/pycore_crossinterp_data_registry.h \ @@ -1360,6 +1363,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_global_strings.h \ $(srcdir)/Include/internal/pycore_hamt.h \ $(srcdir)/Include/internal/pycore_hashtable.h \ + $(srcdir)/Include/internal/pycore_immutability.h \ $(srcdir)/Include/internal/pycore_import.h \ $(srcdir)/Include/internal/pycore_importdl.h \ $(srcdir)/Include/internal/pycore_index_pool.h \ @@ -1410,6 +1414,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pythread.h \ $(srcdir)/Include/internal/pycore_qsbr.h \ $(srcdir)/Include/internal/pycore_range.h \ + $(srcdir)/Include/internal/pycore_regionref.h \ $(srcdir)/Include/internal/pycore_runtime.h \ $(srcdir)/Include/internal/pycore_runtime_init.h \ $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ diff --git a/Modules/_immutablemodule.c b/Modules/_immutablemodule.c index 94ea460c340526e..01127d1b14a0ed8 100644 --- a/Modules/_immutablemodule.c +++ b/Modules/_immutablemodule.c @@ -8,6 +8,7 @@ #include "Python.h" #include +#include "pycore_cown.h" #include "pycore_object.h" #include "pycore_immutability.h" #include "pycore_critical_section.h" @@ -650,6 +651,32 @@ immutable_exec(PyObject *module) { return -1; } + if (PyModule_AddType(module, &_PyTracingRegion_Type) != 0) { + return -1; + } + if (_PyImmutability_SetFreezable( + (PyObject*)&_PyTracingRegion_Type, _Py_FREEZABLE_YES) < 0) { + return -1; + } + + if (PyModule_AddType(module, &_PyCown_Type) != 0) { + return -1; + } + if (_PyImmutability_SetFreezable((PyObject*)&_PyCown_Type, _Py_FREEZABLE_YES) < 0) { + return -1; + } + + if (PyModule_AddType(module, &_PyRegionref_RefType) != 0) { + return -1; + } + // The type object itself has to be freezable: the close trace freezes the + // type of every object it moves into a region. Individual references are + // marked unfreezable in `regionref___new__()` instead. + if (_PyImmutability_SetFreezable( + (PyObject*)&_PyRegionref_RefType, _Py_FREEZABLE_YES) < 0) { + return -1; + } + if (PyModule_AddIntConstant(module, "FREEZABLE_YES", _Py_FREEZABLE_YES) != 0) { return -1; diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 2aee8b07891c919..fe8dabaf8ac4c7a 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -484,13 +484,18 @@ _interp_call_pack(PyThreadState *tstate, struct interp_call *call, "expected a callable, got %R", func); return -1; } - if (_PyFunction_GetXIData(tstate, func, &call->_preallocated.func) < 0) { - PyObject *exc = _PyErr_GetRaisedException(tstate); - if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) { - _PyErr_SetRaisedException(tstate, exc); - return -1; + // If func is immutable (e.g. frozen), share it directly instead of + // marshaling its code. + if (_PyObject_GetXIDataNoFallback(tstate, func, &call->_preallocated.func) < 0) { + _PyErr_Clear(tstate); + if (_PyFunction_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + _PyErr_SetRaisedException(tstate, exc); + return -1; + } + Py_DECREF(exc); } - Py_DECREF(exc); } call->func = &call->_preallocated.func; // Handle the args. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index c73e79eec243fd6..81f93c8332485bb 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -155,6 +155,11 @@ static PyObject * new_statement_cache(pysqlite_Connection *self, pysqlite_state *state, int maxsize) { + // FIXME(regions): statement cache disabled for testing. Return the connection + // itself (its tp_call creates a fresh statement) so callers of + // statement_cache(sql) bypass the functools.lru_cache wrapper. + return Py_NewRef((PyObject *)self); + PyObject *args[] = { NULL, PyLong_FromLong(maxsize), }; if (args[1] == NULL) { return NULL; diff --git a/Objects/cownobject.c b/Objects/cownobject.c new file mode 100644 index 000000000000000..fc3304d78b82ce8 --- /dev/null +++ b/Objects/cownobject.c @@ -0,0 +1,697 @@ +#include "Python.h" +#include "pymacro.h" + +#include "pycore_ceval.h" // _PyEval_AddPendingCall() +#include "pycore_cown.h" +#include "pycore_immutability.h" +#include "pycore_interp.h" // _PyInterpreterState_LookUpID() +#include "pycore_lock.h" +#include "pycore_time.h" // _PyTime_FromSeconds() + +/* Macro that jumps to error, if the expression `x` does not succeed. */ +#define SUCCEEDS(x) { do { int r = (x); if (r != 0) goto error; } while (0); } + +#define Region_Check(x) Py_IS_TYPE((x), &_PyTracingRegion_Type) + +// The interpreter id 0 is used. This value will be used to indicate that +// no interpreter owns the cown. +#define RELEASED_IPID ((_PyCown_ipid_t)0xff00ff00ff00ff00LL) +#define GC_IPID ((_PyCown_ipid_t)0xffff00ff00ff00ffLL) +#define NO_BLOCKING_TIMEOUT -1 +#define UNSET_THREAD_ID ((_PyCown_ipid_t)0xff00000000000000LL) + +typedef enum CownLockStatus { + COWN_ACQUIRE_ERROR = -1, + COWN_ACQUIRE_FAIL = 0, + COWN_ACQUIRE_SUCCESS = 1 +} CownLockStatus; + +// Cowns rely on the immutability machinery for atomic reference counting: +// PyCown_init() freezes each instance once its initial value is installed. +struct _PyCownObject { + PyObject_HEAD + /* The id of the interpreter that currently owns this cown. + * + * This value may be read from and written to from different threads. + * Only use atomic operations to access this field. + */ + // FIXME(cowns): xFrednet: Make sure that an interpreter releases all + // cowns on destruction. + _PyCown_ipid_t owning_ip; + + /* The id of the thread that unlocked this cown. + * + * This is provided as additional information to users, it is not validated + * or used by this cown implementation. + */ + _PyCown_thread_id_t locking_thread; + + /* The value stored in the cown. This value may be immutable, another cown + * or a region object. + */ + PyObject* value; + + /* A lock used, mainly to support timeouts and queueing for locking. + * All other functions should use `owning_ip` to determine if they can + * access the data or not. + * + * Python's mutexes already implement queueing and timeouts in a good way. + * Later we can role our own, if we need but for not this is better. Note + * that the optional GIL release from the lock should not be used, as it + * doesn't seem to account for waiting threads from different interpreters. + * Therefore, we are responsible for releasing and acquireing the GIL. + */ + PyMutex lock; +}; + +_PyCown_ipid_t _PyCown_ReleasedIpid(void) { + return RELEASED_IPID; +} + +_PyCown_thread_id_t _PyCown_UnsetThreadId(void) { + return UNSET_THREAD_ID; +} + +static _PyCown_ipid_t cown_get_owner(_PyCownObject *obj) { + return _Py_atomic_load_uint64(&obj->owning_ip); +} + +_PyCown_ipid_t _PyCown_Owner(PyObject *cown) { + return cown_get_owner(_PyCownObject_CAST(cown)); +} + +_PyCown_thread_id_t _PyCown_LockingThread(PyObject *cown) { + return _Py_atomic_load_uint64(&_PyCownObject_CAST(cown)->locking_thread); +} + +#define BAIL_UNLESS_OWNED_BY(o, owned_by, result) \ + do {\ + _PyCown_ipid_t owning_ip = cown_get_owner(_PyCownObject_CAST(o)); \ + if (owning_ip != owned_by) { \ + PyErr_Format( \ + PyExc_RuntimeError, \ + "attempted to access a cown owned by %llu from %llu", \ + owning_ip, owned_by); \ + return result; \ + } \ + } while (0); +#define BAIL_UNLESS_OWNED(o, result) BAIL_UNLESS_OWNED_BY(o, _PyCown_ThisInterpreterId(), result) +#define BAIL_UNLESS_OWNED_NULL(o) BAIL_UNLESS_OWNED(o, NULL) + +static int cown_set_value_unchecked(_PyCownObject* self, PyObject* value) { + // Storing a value requires ownership. The exception is the teardown of a + // released cown, which nobody owns and only its last reference can reach. + assert(cown_get_owner(self) == RELEASED_IPID + || cown_get_owner(self) == _PyCown_ThisInterpreterId()); + + // The region is moving out of the cown, so its region references answer to + // the cown's owner from now on. + if (self->value != value && Region_Check(self->value)) { + _PyTracingRegion_SetMetaOwner(self->value, cown_get_owner(self)); + } + + // Update the value + Py_XSETREF(self->value, Py_NewRef(value)); + + // The region is now owned by this cown, so its region references resolve + // through it and follow whoever holds it. + if (Region_Check(value)) { + _PyTracingRegion_SetMetaCown(value, _PyObject_CAST(self)); + } + + return 0; +} + +static int cown_set_value(_PyCownObject* self, PyObject* value) { + BAIL_UNLESS_OWNED(self, -1); + + // Bridge objects are allowed + if (Region_Check(value)) { + return cown_set_value_unchecked(self, value); + } + + // Immutable objects are allowed + if (_Py_IsImmutable(value)) { + return cown_set_value_unchecked(self, value); + } + + // Local objects are forbidden + PyErr_Format( + PyExc_RuntimeError, + "attempted to store a local mutable object in a cown.\n" + "Only regions, cown, and immutable objects are allowed"); + + return -1; +} + +/* Attempt to lock the cown. + * + * Timeout values: + * (-1) => Non-blocking locking + * (0) => Block with no timeout + * (n) => Blocking with timeout + */ +static int cown_lock(_PyCownObject* self, PyTime_t timeout, _PyCown_ipid_t locking_ip, bool has_gil) { + // A blocking time should only be set, if this call holds the GIL + assert(has_gil || timeout == NO_BLOCKING_TIMEOUT); + + // Try to lock the mutex directly, without releasing the GIL first + PyLockStatus r = _PyMutex_LockTimed(&self->lock, 0, _Py_LOCK_DONT_DETACH); + + // The cown is currently owned by something else. Release the GIL and + // wait for the timeout. + if (r != PY_LOCK_ACQUIRED && timeout != NO_BLOCKING_TIMEOUT) { + // Release the GIL + Py_BEGIN_ALLOW_THREADS; + + // Attempt to lock the mutex. This uses a PyMutex for the locking, + // timeout and signal handling. + r = _PyMutex_LockTimed( + &self->lock, + timeout, + _Py_LOCK_DONT_DETACH | _PY_LOCK_HANDLE_SIGNALS + ); + + // Acquire the GIL + Py_END_ALLOW_THREADS; + } + + // The lock was interrupted + if (r == PY_LOCK_INTR) { + return COWN_ACQUIRE_ERROR; + } + + // The lock acquisition failed + if (r == PY_LOCK_FAILURE) { + return COWN_ACQUIRE_FAIL; + } + + // Set the owning_ip to the current interpreter, thereby taking ownership + _PyCown_ipid_t released_value = RELEASED_IPID; + if (!_Py_atomic_compare_exchange_uint64( + &self->owning_ip, + &released_value, + locking_ip) + ) { + // Failed to set owning_ip, this should never happen and points + // to a deeper issue. + PyErr_Format( + PyExc_RuntimeError, + "[BUG] failed to set owner on a locked cown\n" + "Cown: %U", + self + ); + + _PyMutex_Unlock(&self->lock); + return COWN_ACQUIRE_ERROR; + } + + // Set the locking thread. Stored atomically because `_PyCown_LockingThread()` + // reads it from interpreters that do not own the cown. + _Py_atomic_store_uint64( + &self->locking_thread, + has_gil ? _PyCown_ThisThreadId() : UNSET_THREAD_ID); + + if (self->value && Region_Check(self->value)) { + assert(!PyObject_GC_IsTracked(self->value)); + PyObject_GC_Track(self->value); + } + + return COWN_ACQUIRE_SUCCESS; +} + +/* Returns the interpreter id used by cowns. + * + * The caller must hold the GIL. + */ +_PyCown_ipid_t _PyCown_ThisInterpreterId(void) { + _PyCown_ipid_t ip = PyInterpreterState_GetID(PyInterpreterState_Get()); + // This should never happen... if it does... we have a problem... + assert(ip != RELEASED_IPID); + return ip; +} + +/* Returns the thread id used by cowns. + * + * The caller must hold the GIL. + */ +_PyCown_thread_id_t _PyCown_ThisThreadId(void) { + _PyCown_thread_id_t id = PyThreadState_GetID(PyThreadState_Get()); + return id; +} + +static int PyCown_init(_PyCownObject *self, PyObject *args, PyObject *kwds) { + // See if we got a value as a keyword argument + static char *kwlist[] = {"value", NULL}; + PyObject *value = Py_None; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &value)) { + return -1; + } + self->value = Py_None; + + // Init the cown as being acquired by the current interpreter + _PyCown_ipid_t this_ip = _PyCown_ThisInterpreterId(); + _Py_atomic_store_uint64(&self->owning_ip, RELEASED_IPID); + if (cown_lock(self, NO_BLOCKING_TIMEOUT, this_ip, true) != COWN_ACQUIRE_SUCCESS) { + PyErr_Format( + PyExc_RuntimeError, + "Newly created cown couldn't be acquired by interpreter %lld (this)", + this_ip); + return -1; + } + + // Set the cown value using the internal function for full validation + SUCCEEDS(cown_set_value(self, value)); + + // Freeze the cown to enable atomic reference counting for it. + PyObject_GC_UnTrack(self); + SUCCEEDS(_PyImmutability_Freeze(_PyObject_CAST(self))); + + return 0; +error: + return -1; +} + +static int PyCown_traverse(_PyCownObject *self, visitproc _ignore1, void* _ignore2) { + // tp_traverse should never be called on cowns since they're not + // tracked by the GC or in any other GC list. The cown type + // still defines `tp_traverse` to ensure that this is never + // accidentally called. Later we may want to simple remove it + // from the type. + assert(false); + return -1; +} + +static int PyCown_reachable(_PyCownObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); + + // The value is explicitly not visited. Freezing or moving cowns should + // not propagate to the value. + // Py_VISIT(self->value); + + return 0; +} + +static int PyCown_clear(_PyCownObject *self) { + if (_Py_IsImmutable(self->value)) { + Py_CLEAR(self->value); + return 0; + } + + // A mutable value is a region, which may only be dropped by the interpreter + // owning this cown. `PyCown_dealloc` makes sure that this runs there. + cown_set_value_unchecked(self, Py_None); + return 0; +} + +/* Tears the cown down. Only the interpreter owning the cown may run this, see + * `cown_handoff_dealloc`. */ +static void cown_dealloc_owned(_PyCownObject *self) { + // Clearing hands the region off, so no region reference points here any more. + PyCown_clear(self); + PyObject_GC_Del(self); +} + +static int cown_pending_dealloc(void *arg) { + cown_dealloc_owned((_PyCownObject *)arg); + return 0; +} + +/* Hands the teardown to the interpreter owning the cown and returns true, or + * returns false when the caller should tear the cown down itself. + * + * A cown is immutable, so the last reference to it can be dropped by an + * interpreter that never owned it. Its region can not be dropped there: while + * the cown is acquired, the region is reference counted non-atomically and + * tracked in the owner's GC list, so touching it would race with the owner. + * + * The cown itself is handed over as well, instead of only its region, because a + * region reference resolving through this cown borrows the pointer. Freeing the + * cown here would leave that pointer dangling until the scheduled call runs. + * + * A cown waiting for its owner is unreachable: its reference count is zero, it + * supports no weak references, and `_Py_TryIncref_Immutable` refuses to + * resurrect it. It must never be revived either, since a revived cown could be + * released a second time, with its region already gone. + */ +static bool cown_handoff_dealloc(_PyCownObject *self) { + _PyCown_ipid_t owner = cown_get_owner(self); + // Nobody owns a released cown, which makes the caller the only one that can + // reach the region. + if (owner == RELEASED_IPID || owner == _PyCown_ThisInterpreterId()) { + return false; + } + + // The lookup raises when the interpreter is gone, and a deallocation can + // happen mid-raise. + PyObject *exc = PyErr_GetRaisedException(); + // FIXME(regions): Can the interpreter go away in the middle of scheduling? + // `weakref_schedule_callbacks` in `Python/immutability.c` asks the same. + PyInterpreterState *target = _PyInterpreterState_LookUpID((int64_t)owner); + bool scheduled = target != NULL + && _PyEval_AddPendingCall(target, cown_pending_dealloc, self, 0) + == _Py_ADD_PENDING_SUCCESS; + PyErr_SetRaisedException(exc); + if (scheduled) { + return true; + } + + // The owner is gone, or its call queue is full. Tearing the cown down here + // is all that is left to do, so the region ends up owned by nobody. An + // interpreter that is already gone can at least not race with us. + _Py_atomic_store_uint64(&self->owning_ip, RELEASED_IPID); + return false; +} + +static void PyCown_dealloc(_PyCownObject *self) { + // Reaching zero returned the cown to this interpreter's GC list. Nothing may + // traverse it, `PyCown_traverse` asserts as much. + PyObject_GC_UnTrack(self); + + if (cown_handoff_dealloc(self)) { + return; + } + cown_dealloc_owned(self); +} + +static int +lock_acquire_parse_args(PyObject *args, PyObject *kwds, + PyTime_t *timeout) +{ + // Taken from `Modules/_threadmodule.c` + + char *kwlist[] = {"blocking", "timeout", NULL}; + int blocking = 1; + PyObject *timeout_obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pO:acquire", kwlist, + &blocking, &timeout_obj)) + return -1; + + const PyTime_t unset_timeout = _PyTime_FromSeconds(NO_BLOCKING_TIMEOUT); + *timeout = unset_timeout; + + if (timeout_obj + && _PyTime_FromSecondsObject(timeout, + timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) + return -1; + + if (!blocking && *timeout != unset_timeout ) { + PyErr_SetString(PyExc_ValueError, + "can't specify a timeout for a non-blocking call"); + return -1; + } + if (*timeout < 0 && *timeout != unset_timeout) { + PyErr_SetString(PyExc_ValueError, + "timeout value must be a non-negative number"); + return -1; + } + if (!blocking) + *timeout = 0; + else if (*timeout != unset_timeout) { + PyTime_t microseconds; + + microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_TIMEOUT); + if (microseconds > PY_TIMEOUT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "timeout value is too large"); + return -1; + } + } + return 0; +} + +static PyObject * +CownObject_acquire(_PyCownObject *self, PyObject *args, PyObject *kwds) +{ + // Parse the arguments + PyTime_t timeout; + if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { + return NULL; + } + + // Attempt to lock the cown + _PyCown_ipid_t this_ip = _PyCown_ThisInterpreterId(); + int res = cown_lock(self, timeout, this_ip, true); + if (res == COWN_ACQUIRE_ERROR) { + return NULL; + } + + // Return the result + return PyBool_FromLong(res == COWN_ACQUIRE_SUCCESS); +} + +PyDoc_STRVAR(CownObject_acquire_doc, +"acquire($self, /, blocking=True, timeout=-1)\n\ +--\n\ +\n\ +Attempts to acquires the cown. With default arguments this will block\n\ +until the cown can be aquired, even when acquire is called from the same\n\ +interpreter. The return indicates if the cown was\n\ +was acquired. The blocking operation is interruptible."); + +static int cown_release_unchecked(_PyCownObject* self, _PyCown_ipid_t unlocking_ip) { + // Set owning_ip to indicate the released state + if (!_Py_atomic_compare_exchange_uint64(&self->owning_ip, &unlocking_ip, RELEASED_IPID)) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %lld (this) attempted to release a cown owned by someone else\n" + "Cown: %U", + unlocking_ip, self); + return -1; + } + + // Unlocking should always succeed + int res = _PyMutex_TryUnlock(&self->lock); + assert(res == 0); + (void)res; + + return 0; +} + +/* Checks that the cown is not released, and that the owner is as the current interpreter. */ +static int cown_check_owner_before_release(_PyCownObject *self, _PyCown_ipid_t unlocking_ip) { + _PyCown_ipid_t owning_ip = cown_get_owner(self); + if (owning_ip == RELEASED_IPID) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %lld attempted to release/switch a released cown", + unlocking_ip + ); + return -1; + } + if (owning_ip != unlocking_ip) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %lld attempted to release/switch a cown owned by %lld", + unlocking_ip, owning_ip + ); + return -1; + } + return 0; +} + +/* This attempts to close the region + * + * It returns non-zero if the closing failed + */ +static int cown_close_region(_PyCownObject *self) { + assert(Region_Check(self->value)); + + // Close the region + int closing_res = _PyTracingRegion_Close(self->value); + if (closing_res < 0) { + return -1; + } + + // Make sure that the cown owns the only external reference to the bridge object. + if (Py_REFCNT(self->value) > 1) { + PyErr_Format( + PyExc_RuntimeError, + "the cown couldn't be released, due to the bridge having incoming references"); + return -1; + } + + // The region is closed and this is the only owner of the bridge. We untrack + // from the current GC list. + PyObject_GC_UnTrack(self->value); + + return 0; +} + +static int cown_release(_PyCownObject *self, _PyCown_ipid_t unlocking_ip) { + if (cown_check_owner_before_release(self, unlocking_ip) < 0) { + return -1; + } + + // Immutable objects are safe to share, the cown can be release directly + if (_Py_IsImmutable(self->value)) { + return cown_release_unchecked(self, unlocking_ip); + } + assert(Region_Check(self->value)); + + // The contained region needs to be closed, to allow the cown to release + if (cown_close_region(self)) { + return -1; + } + + // The close leaves the region local to this interpreter. Rooting it here, + // after every check has passed, is what lets the next owner of the cown + // dereference the region references pointing into it. + _PyTracingRegion_SetMetaCown(self->value, _PyObject_CAST(self)); + + // Region is closed, safe to release + return cown_release_unchecked(self, unlocking_ip); +} + +static PyObject* CownObject_release(_PyCownObject *self, PyObject *ignored) { + _PyCown_ipid_t this_ip = _PyCown_ThisInterpreterId(); + if (cown_release(self, this_ip) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(CownObject_release_doc, +"release($self, /)\n\ +--\n\ +\n\ +Release the cown, allowing another interpreter that is blocked waiting for\n\ +the cown to acquire the cown. The cown must be in the locked state\n\ +and must be unlocked from the owning interpreter. It may be unlocked \n\ +by any thread on the owning interpreter."); + +static PyObject * +CownObject_locked(_PyCownObject *op, PyObject *Py_UNUSED(dummy)) +{ + return PyBool_FromLong(cown_get_owner(op) != RELEASED_IPID); +} + +PyDoc_STRVAR(CownObject_locked_doc, +"locked($self, /)\n\ +--\n\ +\n\ +Return whether the cown currently released or aquired. \n\ +Use `owned()` to check if the cown is aquired by the current interpreter."); + +static PyObject * +CownObject_owned(_PyCownObject *op, PyObject *Py_UNUSED(dummy)) +{ + return PyBool_FromLong(cown_get_owner(op) == _PyCown_ThisInterpreterId()); +} + +PyDoc_STRVAR(CownObject_owned_doc, +"owned($self, /)\n\ +--\n\ +\n\ +Return true if the cown is currently aquired by this interpreter, false otherwise."); + +static PyObject * +CownObject_owned_by_thread(_PyCownObject *op, PyObject *Py_UNUSED(dummy)) +{ + if (cown_get_owner(op) != _PyCown_ThisInterpreterId()) { + Py_RETURN_FALSE; + } + + return PyBool_FromLong(op->locking_thread == _PyCown_ThisThreadId()); +} + +PyDoc_STRVAR(CownObject_owned_by_thread_doc, +"owned($self, /)\n\ +--\n\ +\n\ +Return true if the cown is currently aquired by this interpreter and was \n\ +locked by the current thread, false otherwise. \n\ +Ownership on the thread level is not enforced, any thread on the owning\n\ +interpreter can access and release the cown. This is information is only\n\ +provided to give more control for those who seek it."); + +static PyObject * +CownObject_is_closed(_PyCownObject *self, PyObject *Py_UNUSED(dummy)) +{ + if (!Region_Check(self->value)) { + PyErr_SetString(PyExc_TypeError, "cown value is not a tracing region"); + return NULL; + } + + return PyBool_FromLong(_PyTracingRegion_IsClosed(self->value)); +} + +PyDoc_STRVAR(CownObject_is_closed_doc, +"_is_closed($self, /)\n\ +--\n\ +\n\ +Return true if the cown's tracing region value is closed."); + + +// Define the CownType with methods +static PyMethodDef PyCown_methods[] = { + {"acquire", _PyCFunction_CAST(CownObject_acquire), METH_VARARGS | METH_KEYWORDS, CownObject_acquire_doc}, + {"release", _PyCFunction_CAST(CownObject_release), METH_NOARGS, CownObject_release_doc}, + {"locked", _PyCFunction_CAST(CownObject_locked), METH_NOARGS, CownObject_locked_doc}, + {"owned", _PyCFunction_CAST(CownObject_owned), METH_NOARGS, CownObject_owned_doc}, + {"owned_by_thread", _PyCFunction_CAST(CownObject_owned_by_thread), METH_NOARGS, CownObject_owned_by_thread_doc}, + {"_is_closed", _PyCFunction_CAST(CownObject_is_closed), METH_NOARGS, CownObject_is_closed_doc}, + {NULL} // Sentinel +}; + +static PyObject *CownObject_get_value(_PyCownObject *self, void *closure) { + BAIL_UNLESS_OWNED_NULL(self); + + return Py_NewRef(self->value); +} + +static int CownObject_set_value(_PyCownObject *self, PyObject *value, void *closure) { + BAIL_UNLESS_OWNED(self, -1); + + return cown_set_value(self, value); +} + +static PyGetSetDef PyCownObject_getset[] = { + {"value", (getter)CownObject_get_value, (setter)CownObject_set_value, + "", NULL}, + {NULL, NULL, NULL, NULL, NULL} +}; + +static PyObject *PyCown_repr(_PyCownObject *self) { + _PyCown_ipid_t owner = cown_get_owner(self); + // On this interpreter we can access the cown and content + // safely since we hold the GIL + if (owner == _PyCown_ThisInterpreterId()) { + return PyUnicode_FromFormat( + "Cown(interpreter=%llu (this), value=%S)", + owner, + PyObject_Repr(self->value) + ); + } + + // The cown is released and can be acquired + if (owner == RELEASED_IPID) { + return PyUnicode_FromFormat( + "Cown(interpreter=None, status=Released)" + ); + } + + // The cown is owned by a different interpreter + return PyUnicode_FromFormat( + "Cown(interpreter=%llu (other))", + owner + ); +} + +PyTypeObject _PyCown_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "Cown", + .tp_basicsize = sizeof(_PyCownObject), + .tp_dealloc = (destructor)PyCown_dealloc, + .tp_repr = (reprfunc)PyCown_repr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .tp_traverse = (traverseproc)PyCown_traverse, + .tp_reachable = (traverseproc)PyCown_reachable, + .tp_clear = (inquiry)PyCown_clear, + .tp_methods = PyCown_methods, + .tp_getset = PyCownObject_getset, + .tp_init = (initproc)PyCown_init, + .tp_new = PyType_GenericNew, +}; + diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c new file mode 100644 index 000000000000000..ee8c7c90028c390 --- /dev/null +++ b/Objects/tracingregionobject.c @@ -0,0 +1,1973 @@ +#include "Python.h" +#include "pycore_interp.h" +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_dict.h" // _PyObject_MaterializeManagedDict() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() +#include "pycore_descrobject.h" +#include "pycore_modsupport.h" // _PyArg_NoPositional() +#include "pycore_weakref.h" +#include "pycore_cown.h" +#include "pycore_regionref.h" + +#define ERROR_OBJECT_REPORT_COUNT 5 +#define ERROR_MERMAID_REPORT_LIMIT 50 +#define ERROR_MERMAID_HIDE_IMMUTABLE true + +/* Set this to the path of the file that a failed close should write its mermaid + * graph to. The graph is not written when the variable is unset or empty. */ +#define REGION_GRAPH_ENV_VAR "PYTHON_REGION_GRAPH" + +#define REGION_TRACING + +#ifdef REGION_TRACING +#define dbg(msg, ...) \ + do { \ + printf(msg "\n" __VA_OPT__(,) __VA_ARGS__); \ + } while(0) +#else +#define dbg(...) +#endif + +/* Macro that jumps to error, if the expression `x` does not succeed. */ +#define SUCCEEDS(x) do { int r = (x); if (r != 0) goto error; } while (0) + +#define Region_Check(x) Py_IS_TYPE((x), &_PyTracingRegion_Type) +#define Cown_Check(x) Py_IS_TYPE((x), &_PyCown_Type) + +// ################################################################### +// Copied from gc.c +// ################################################################### + +#ifndef Py_GIL_DISABLED +#define GC_NEXT _PyGCHead_NEXT +#define GC_PREV _PyGCHead_PREV + +static inline int +gc_old_space(PyGC_Head *g) +{ + return g->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1; +} + +static inline void +gc_set_old_space(PyGC_Head *g, int space) +{ + assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1); + g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1; + g->_gc_next |= space; +} + +static inline void +gc_list_init(PyGC_Head *list) +{ + // List header must not have flags. + // We can assign pointer by simple cast. + list->_gc_prev = (uintptr_t)list; + list->_gc_next = (uintptr_t)list; +} + +static void +gc_list_move(PyGC_Head *node, PyGC_Head *list) +{ + /* Unlink from current list. */ + PyGC_Head *from_prev = GC_PREV(node); + PyGC_Head *from_next = GC_NEXT(node); + _PyGCHead_SET_NEXT(from_prev, from_next); + _PyGCHead_SET_PREV(from_next, from_prev); + + /* Relink at end of new list. */ + // list must not have flags. So we can skip macros. + PyGC_Head *to_prev = (PyGC_Head*)list->_gc_prev; + _PyGCHead_SET_PREV(node, to_prev); + _PyGCHead_SET_NEXT(to_prev, node); + list->_gc_prev = (uintptr_t)node; + _PyGCHead_SET_NEXT(node, list); +} + +static inline int +gc_list_is_empty(PyGC_Head *list) +{ + return (list->_gc_next == (uintptr_t)list); +} + +static void +gc_list_merge(PyGC_Head *from, PyGC_Head *to) +{ + assert(from != to); + if (!gc_list_is_empty(from)) { + PyGC_Head *to_tail = GC_PREV(to); + PyGC_Head *from_head = GC_NEXT(from); + PyGC_Head *from_tail = GC_PREV(from); + assert(from_head != from); + assert(from_tail != from); + assert(gc_list_is_empty(to) || + gc_old_space(to_tail) == gc_old_space(from_tail)); + + _PyGCHead_SET_NEXT(to_tail, from_head); + _PyGCHead_SET_PREV(from_head, to_tail); + + _PyGCHead_SET_NEXT(from_tail, to); + _PyGCHead_SET_PREV(to, from_tail); + } + gc_list_init(from); +} + +static struct _gc_runtime_state* +get_gc_state(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; +} + +static inline void +gc_clear_collecting(PyGC_Head *g) +{ + g->_gc_prev &= ~_PyGC_PREV_MASK_COLLECTING; +} + +#else // Py_GIL_DISABLED +#error "We need GIL" +#endif + +// ################################################################### +// Copied from regions-main +// ################################################################### + +/* Removes the last item of the list and returns it as a new reference. + * + * The caller needs a reference of its own, since the list was the only thing + * keeping the item alive. Traversing the item can run arbitrary code, for + * example through `_PyImmutability_Freeze()`, which could otherwise deallocate + * it while it is being traversed. + * + * Returns NULL with an exception set on failure. The list must not be empty. + */ +static PyObject* list_pop(PyObject* s){ + Py_ssize_t size = PyList_GET_SIZE(s); + assert(size > 0); + + PyObject *item = Py_NewRef(PyList_GET_ITEM(s, size - 1)); + // This should never fail, since we shrink the size + if (PyList_SetSlice(s, size - 1, size, NULL)) { + Py_DECREF(item); + return NULL; + } + return item; +} + +typedef enum { + Py_MOVABLE_YES = 0, + Py_MOVABLE_NO = 1, + // The object should be frozen + Py_MOVABLE_FREEZE = 2, + // The object is not movable, but the reference is allowed. The object + // should be skipped + Py_MOVABLE_COWN = 3, +} movable_status; + +static movable_status get_movable_status(PyObject *obj) { + // FIXME(regions): xFrednet: Currently it's not possible to set + // the movability per object. This instead returns the default + // movability for objects. Note that some shallow immutable objects + // will not return freeze as their movability. + + // Immortal object have no real RC, this makes it infeasible to have them + // in a region and dynamically track their ownership. Immortal objects are + // intended to be immutable in Python, so it should be safe to implicitly + // freeze them. + if (_Py_IsImmortal(obj)) { + return Py_MOVABLE_FREEZE; + } + + // Immutable objects don't need to be moved + if (_Py_IsImmutable(obj)) { + return Py_MOVABLE_FREEZE; + } + + // Types are a pain for regions since it's likely that objects of one type may + // end up in multiple regions, requiring the type to be frozen. Types also + // have a lot of reference pointing to them. Let's hope there is no need to + // keep them freezable + if (PyType_Check(obj)) { + return Py_MOVABLE_FREEZE; + } + + // Module objects are also complicated. Freezing them should turn most modules + // into proxies which should make them mostly usable. + if (PyModule_Check(obj)) { + return Py_MOVABLE_FREEZE; + } + + // Functions are a mess as well, making the entire system reachable. Freezing + // them should again just magically make most things work + if (PyFunction_Check(obj)) { + return Py_MOVABLE_FREEZE; + } + + // CWrappers can't really be owned, but need some special handling since + // interpreters could still race on their RC. Solution, throw them in the + // freezer + if (PyCFunction_Check(obj) + || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) + || Py_IS_TYPE(obj, &PyWrapperDescr_Type) + ) { + return Py_MOVABLE_FREEZE; + } + + // Cowns are not movable, but the reference is explicitly allowed. + if (Cown_Check(obj)) { + return Py_MOVABLE_COWN; + } + + // Freezing or moving these objects is... complicated. In some cases it is + // possible but more hassle than it's probably worth. For now we mark them + // all as unmovable. + if (PyFrame_Check(obj) + || PyGen_CheckExact(obj) + || PyCoro_CheckExact(obj) + || PyAsyncGen_CheckExact(obj) + || PyAsyncGenASend_CheckExact(obj) + ) { + return Py_MOVABLE_NO; + } + + // Exceptions don't hold anything obviously problematic preventing them + // from being moved into a region. The actual problem is that the runtime + // stores references to them and that these are already emitted on an + // error path. Moving them into a region could add more problems. + // We should discuss how to handle these, maybe freezing is the correct + // approach? + if (PyExceptionInstance_Check(obj)) { + return Py_MOVABLE_NO; + } + + // Regions are theoretically only movable, if they're closed. The traversal + // checks this manually. + + // For now, we define all other objects as movable by default. (Surely + // this will not backfire) + return Py_MOVABLE_YES; +} + +// This uses the given arguments to create and throw a `RuntimeError` +static void throw_region_error( + const char *format_str, const char *tp_name, + PyObject* src, PyObject* tgt) +{ + // Don't stomp existing exception + PyThreadState *tstate = PyThreadState_Get(); + if (_PyErr_Occurred(tstate)) { + return; + } + + PyErr_Format(PyExc_RuntimeError, format_str, tp_name); + + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + + // Failing to attach it must not replace the error raised above. + if (PyObject_SetAttr(exc, &_Py_ID(source), src ? src : Py_None) < 0 + || PyObject_SetAttr(exc, &_Py_ID(target), tgt ? tgt : Py_None) < 0) + { + PyErr_Clear(); + } + + PyErr_SetRaisedException(exc); +} + +// Wrapper around tp_traverse that also visits the type object. +static int +traverse_via_tp_traverse(PyObject *obj, visitproc visit, void *state) +{ + PyTypeObject *tp = Py_TYPE(obj); + + // Visit the type with traverse + traverseproc traverse = tp->tp_traverse; + if (traverse != NULL) { + int err = traverse(obj, visit, state); + if (err) { + return err; + } + } + + // Most `tp_traverse` don't visit the type even though they should. + // Here it won't hurt to potentially visit it twice, since types + // are non-movable but will be frozen. + return visit((PyObject *)tp, state); +} + +/* Returns the appropriate traversal function for reaching all references from + * an object. Prefers tp_reachable, falls back to tp_traverse wrapped to also + * visit the type. + * + * Falling back means the trace can miss references that only tp_reachable + * reports, so every type it happens for is recorded in `missing_reachable` and + * reported by `report_missing_reachable()` once the trace is over. Warning here + * would write to `sys.stderr` in the middle of the traversal, which can run + * arbitrary Python code and invalidate the reference counts already sampled. + * + * `missing_reachable` may be NULL to skip the recording. + */ +static traverseproc +get_reachable_proc(PyTypeObject *tp, _Py_hashtable_t *missing_reachable) +{ + if (tp->tp_reachable != NULL) { + return tp->tp_reachable; + } + + if (missing_reachable != NULL + && _Py_hashtable_get_entry(missing_reachable, tp) == NULL) + { + // Types are frozen rather than moved, so `_move_obj()` returns before it + // samples their reference count. Holding one here can therefore not + // disturb the LRC of any region. + if (_Py_hashtable_set(missing_reachable, Py_NewRef(tp), + (void *)(Py_uintptr_t)(tp->tp_traverse != NULL)) < 0) { + Py_DECREF(tp); + // A failed warning must not fail the close. + PyErr_Clear(); + } + } + + // Always return the wrapper; even when tp_traverse is NULL, the wrapper + // will still visit the type object which tp_reachable is expected to do. + return traverse_via_tp_traverse; +} + +static int +report_missing_reachable_type( + _Py_hashtable_t *ht, const void *key, const void *value, void *user_data) +{ + PyTypeObject *tp = (PyTypeObject *)key; + if (value) { + PySys_FormatStderr( + "regions: type '%.100s' has tp_traverse but no tp_reachable\n", + tp->tp_name); + } + else { + PySys_FormatStderr( + "regions: type '%.100s' has no tp_traverse and no tp_reachable\n", + tp->tp_name); + } + return 0; +} + +static int +release_missing_reachable_type( + _Py_hashtable_t *ht, const void *key, const void *value, void *user_data) +{ + Py_DECREF((PyObject *)key); + return 0; +} + +// ################################################################### +// Tracing Impl +// ################################################################### + +static void +gc_list_dissolve(PyGC_Head *list) { + struct _gc_runtime_state* gc_state = get_gc_state(); + gc_list_merge(list, &(gc_state->old[0].head)); +} + +typedef struct { + // The weak references that live inside the region and therefore survive the + // close. NULL when the trace did not find any. + _Py_hashtable_t *keep; + PyObject *region; +} detach_weak_refs_state_t; + +static int +detach_weak_refs_visit(_Py_hashtable_t *ht, const void *key, const void *value, void *user_data) +{ + PyObject *item = (PyObject *)key; + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(item))) { + return 0; + } + detach_weak_refs_state_t *state = (detach_weak_refs_state_t *)user_data; + +#ifdef Py_DEBUG + Py_ssize_t weak_ctn = _PyWeakref_GetWeakrefCount(item); + if (weak_ctn) { + dbg("- Clearing %zd weak references to %p", weak_ctn, item); + } +#endif + _PyRegionRef_CloseWeakRefs(item, state->keep, state->region); + return 0; +} + +/* Detaches all weak references pointing to objects inside the region, and + * re-homes the region references, which are meant to survive the close. + * + * This walks the set of traced objects instead of the region's GC list, since + * objects that are not tracked by the GC never enter that list. Missing one + * would leave a live weak reference pointing into the closed region, which is + * enough for external code to read and mutate its contents. + * + * Re-homing rides along on this walk on purpose. Finding the references into a + * region means looking at every member's weakref list, which is exactly what + * this already does. + */ +static void detach_weak_refs( + PyObject *region, _Py_hashtable_t *visited, bool has_weak_refs) +{ + detach_weak_refs_state_t state = { + .keep = has_weak_refs ? visited : NULL, + .region = region, + }; + // `detach_weak_refs_visit()` never fails, so the result can be ignored. + (void)_Py_hashtable_foreach(visited, detach_weak_refs_visit, &state); +} + +typedef struct { + PyObject_HEAD + PyObject *dict; + // The GC list containing all objects while the region is closed. The bridge + // object is not in this GC list but in the list of the owning region or in no + // list if it's owned by a released cown. + PyGC_Head gc_list; + // All objects that belong to a closed region are in the `gc_list` above. This + // removes them from the local GC and allows this region to be moved between + // sub-interpreters, but it would prevent the collection of closed regions with + // internal references to the bridge. On closed regions, we therefore manually + // subtract internal references from the RC. We basically hide the cycles, until + // the region is open. This is the number of references subtracted from the rc. + // These are readded in the constructor or when opening the region. + Py_ssize_t internal_bridge_refs; + // The node every region reference into this region resolves through, or + // NULL when nothing points into it. Only closed regions can have a meta, + // opening restamps it + _PyRegionRefMetadata *meta; + // FIXME(regions): This can be inferred from the status of the gc_list + // or stored in the lower bits of the GC list. For now we keep it separate + // for the prototype + bool open; +} TracingRegionObject; + +/* Returns this region's node, allocating it if this is the first reference the + * current close has found. Borrowed. The caller must hold `_PyWeakref_Lock`. */ +static _PyRegionRefMetadata * +region_meta_lock_held(TracingRegionObject *self) +{ + if (self->meta == NULL) { + self->meta = _PyRegionRef_NewRegionMetaLockHeld(_PyObject_CAST(self)); + } + return self->meta; +} + +/* Hands the node over to the references still holding it: it becomes local to + * this interpreter with nothing left to open. Used whenever a region stops + * being closed, including when its contents are being deleted. */ +static void +region_meta_release(TracingRegionObject *self) +{ + if (self->meta == NULL) { + return; + } + _PyRegionRef_MetaRegionOpened(self->meta); + _PyRegionRef_MetaDecref(self->meta); + self->meta = NULL; +} + +static void _region_close( + TracingRegionObject *self, + Py_ssize_t bridge_rc, + _Py_hashtable_t *visited, + bool has_weak_refs +) { + if (!self->open) { + return; + } + + dbg("Closing region %p", self); + + detach_weak_refs(_PyObject_CAST(self), visited, has_weak_refs); + + // See comment on `self->internal_bridge_refs` + if (bridge_rc != 0) { + assert(bridge_rc >= 0); + dbg("- subtracting %zd internal references from the bridge object %p", bridge_rc, self); + _Py_RefcntAdd(self, -bridge_rc); + self->internal_bridge_refs = bridge_rc; + } else { + assert(self->internal_bridge_refs == 0); + } + + self->open = false; +} + +/* Re-adds the references to the bridge object that `_region_close()` subtracted. + * + * Note that this may resurrect the bridge object. Callers may need to handle this case. + */ +static void _restore_internal_bridge_refs(TracingRegionObject *self) { + if (self->internal_bridge_refs != 0) { + assert(self->internal_bridge_refs >= 0); + dbg("- adding %zd internal references from the bridge object %p", self->internal_bridge_refs, self); + _Py_RefcntAdd(self, self->internal_bridge_refs); + self->internal_bridge_refs = 0; + } +} + +static void _open_region(TracingRegionObject *self) { + if (self->open) { + return; + } + + dbg("Opening region %p", self); + + region_meta_release(self); + _restore_internal_bridge_refs(self); + + // This only dissolves this region, all sub-regions remain closed. + gc_list_dissolve(&self->gc_list); + assert(gc_list_is_empty(&self->gc_list)); + + self->open = true; +} + +#define PER_REGION_TRACE_LIMIT 2 + +typedef struct { + // This is the stack of regions that still need to be closed to close this + // region tree. A region stays on the stack until it is closed, so anything + // its trace discovers is pushed on top of it and handled first. The loop can + // therefore only drain once every region in the tree is closed. + // + // How many attempts a region gets is tracked by `tracing_counts`. + PyObject *pending; + // This tracks per region in the tree how often it has been traversed. + // Some things require the trace to be redone, namely freezing an object + // as that may create references and finding an open sub-region, as that + // one needs to be traced and closed first. + // + // We limit the number of times we restart the trace per region. + // Theoretically, this may reject some programs that would eventually + // reach a fixed point, but if somebody wants to do dark magic, that's + // really not our problem. + _Py_hashtable_t *tracing_counts; + // The types that had to be traversed via tp_traverse because they have no + // tp_reachable. Used to report each of them once per trace, see + // `get_reachable_proc()`. + _Py_hashtable_t *missing_reachable; + // The region hierarchy of this trace, child nodes map to their parents. + _Py_hashtable_t *hierarchy; +} tree_trace_state_t; + +static void tree_trace_state_destroy(tree_trace_state_t* state) { + if (state->tracing_counts) { + _Py_hashtable_destroy(state->tracing_counts); + state->tracing_counts = NULL; + } + if (state->missing_reachable) { + (void)_Py_hashtable_foreach( + state->missing_reachable, release_missing_reachable_type, NULL); + _Py_hashtable_destroy(state->missing_reachable); + state->missing_reachable = NULL; + } + if (state->hierarchy) { + _Py_hashtable_destroy(state->hierarchy); + state->hierarchy = NULL; + } + if (state->pending) { + Py_CLEAR(state->pending); + } +} + +/* Reports the types that `get_reachable_proc()` had to fall back for. + * + * This has to run after the traversal is over, since writing to `sys.stderr` + * can execute arbitrary Python code. + */ +static void report_missing_reachable(tree_trace_state_t* state) { + if (state->missing_reachable == NULL + || _Py_hashtable_len(state->missing_reachable) == 0) + { + return; + } + + // Keep whatever the trace is raising; a failed warning is not worth + // replacing a region error with. + PyObject *exc = PyErr_GetRaisedException(); + (void)_Py_hashtable_foreach( + state->missing_reachable, report_missing_reachable_type, NULL); + PyErr_SetRaisedException(exc); +} + +static int tree_trace_state_init(tree_trace_state_t* state) { + // Both fields have to be cleared up front, so that the error path below can + // call `tree_trace_state_destroy()` before they have all been assigned. + state->tracing_counts = NULL; + state->missing_reachable = NULL; + state->pending = NULL; + state->hierarchy = NULL; + + state->tracing_counts = _Py_hashtable_new( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + if (state->tracing_counts == NULL) { + PyErr_NoMemory(); + goto error; + } + + state->missing_reachable = _Py_hashtable_new( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + if (state->missing_reachable == NULL) { + PyErr_NoMemory(); + goto error; + } + + state->pending = PyList_New(0); + if (state->pending == NULL) { + goto error; + } + + state->hierarchy = _Py_hashtable_new( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + if (state->hierarchy == NULL) { + PyErr_NoMemory(); + goto error; + } + + return 0; +error: + tree_trace_state_destroy(state); + return -1; +} + +typedef struct { + // List of pending objects that are not GC + PyObject *pending; + // A list of all visited objects + _Py_hashtable_t *visited; + + // The trace state belonging to the region tree that this region + // is a part of. + tree_trace_state_t *tree_trace_state; + // The bridge object of the region that is currently being traced. + PyObject* bridge; + // The source of the reference, this is used for error reporting + PyObject *src; + + // The number of refs coming into this object graph + Py_ssize_t external_rc; + // The number of refs coming from inside the region to the bridge object + Py_ssize_t bridge_rc; + + // The GC list used for this trace, it may be null if the trace + // should not move the objects from their current list. + PyGC_Head* gc_list; + + + // This is set if an object was frozen and the trace needs + // to restart to be valid + bool restart; + + // Indicates if the given reference is a strong reference or a weak one. + bool strong_ref; + + bool has_weak_refs; +} region_trace_state_t; + +static void region_trace_state_destroy(region_trace_state_t* state) { + if (state->pending) { + Py_CLEAR(state->pending); + } + if (state->visited) { + _Py_hashtable_destroy(state->visited); + state->visited = NULL; + } +} + +static int region_trace_state_init( + region_trace_state_t* state, + PyObject* bridge, + PyGC_Head* gc_list, + tree_trace_state_t *tree_trace_state +) { + assert(gc_list == NULL || gc_list_is_empty(gc_list)); + + state->pending = NULL; + state->visited = NULL; + + state->pending = PyList_New(0); + if (state->pending == NULL) { + goto error; + } + + state->visited = _Py_hashtable_new( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + if (state->visited == NULL) { + goto error; + } + + state->tree_trace_state = tree_trace_state; + state->bridge = bridge; + state->src = NULL; + + state->external_rc = 0; + state->bridge_rc = 0; + state->gc_list = gc_list; + state->restart = false; + // References are strong unless the trace explicitly follows a weak one. + state->strong_ref = true; + state->has_weak_refs = false; + + return 0; +error: + region_trace_state_destroy(state); + return -1; +} + +static void region_trace_state_set_restart(region_trace_state_t* state) { + state->restart = true; + // Setting the gc_list to NULL will stop objects from being moved + // between GC lists. Just a small thing we can avoid. The next (full) + // trace will have this set again. + state->gc_list = NULL; +} + +typedef struct { + // Every object with incoming references, used to mark up the mermaid graph. + _Py_hashtable_t *problem_objs; + // The subset of `problem_objs` that the error message lists, capped at + // `ERROR_OBJECT_REPORT_COUNT` entries. + _Py_hashtable_t *reported_objs; + Py_ssize_t incoming_refs; +} close_error_info_t; + +typedef struct { + _Py_hashtable_t *problem_objs; + _Py_hashtable_t *reported_objs; +} close_error_filter_t; + +typedef struct { + // A strong reference, see `collect_incoming_ref()`. + PyObject *obj; + Py_ssize_t refs; +} incoming_ref_entry_t; + +typedef struct { + // `collect_close_error_obj()` caps the reported set at this size. + incoming_ref_entry_t entries[ERROR_OBJECT_REPORT_COUNT]; + Py_ssize_t count; +} incoming_ref_report_t; + +typedef struct { + PyUnicodeWriter *writer; + _Py_hashtable_t *visited; + _Py_hashtable_t *problem_objs; + _Py_hashtable_t *reported_objs; + PyObject *pending; + PyObject *src; +} mermaid_dump_state_t; + +enum { + TRACE_RES_ERR = -1, + TRACE_RES_DONE = 0, + // The trace itself succeeded, but it was based on information that changed + // while it ran, so the region is still open and needs another attempt. + TRACE_RES_RESTART = 1, +}; + +static int +collect_close_error_obj(_Py_hashtable_t *ht, const void *key, const void *value, void *user_data) +{ + close_error_filter_t *filter = (close_error_filter_t *)user_data; + Py_ssize_t refs = (Py_ssize_t)value; + + // Objects whose every reference came from inside the region are not part of + // the problem. + if (refs <= 0) { + return 0; + } + if (_Py_hashtable_set(filter->problem_objs, key, (void *)refs) < 0) { + PyErr_NoMemory(); + return -1; + } + if (_Py_hashtable_len(filter->reported_objs) < ERROR_OBJECT_REPORT_COUNT) { + if (_Py_hashtable_set(filter->reported_objs, key, (void *)refs) < 0) { + PyErr_NoMemory(); + return -1; + } + } + return 0; +} + +static void +close_error_info_destroy(close_error_info_t *info) +{ + if (info->problem_objs != NULL) { + _Py_hashtable_destroy(info->problem_objs); + info->problem_objs = NULL; + } + if (info->reported_objs != NULL) { + _Py_hashtable_destroy(info->reported_objs); + info->reported_objs = NULL; + } +} + +static int +close_error_info_init(close_error_info_t *info, region_trace_state_t *state) +{ + info->incoming_refs = state->external_rc; + info->problem_objs = NULL; + info->reported_objs = NULL; + info->problem_objs = _Py_hashtable_new( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + if (info->problem_objs == NULL) { + return -1; + } + info->reported_objs = _Py_hashtable_new( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + if (info->reported_objs == NULL) { + close_error_info_destroy(info); + return -1; + } + + close_error_filter_t filter = {info->problem_objs, info->reported_objs}; + int res = _Py_hashtable_foreach(state->visited, collect_close_error_obj, &filter); + if (res < 0) { + close_error_info_destroy(info); + return -1; + } + return 0; +} + +static int +collect_incoming_ref(_Py_hashtable_t *ht, const void *key, const void *value, void *user_data) +{ + incoming_ref_report_t *report = (incoming_ref_report_t *)user_data; + + assert(report->count < ERROR_OBJECT_REPORT_COUNT); + if (report->count >= ERROR_OBJECT_REPORT_COUNT) { + return 0; + } + + incoming_ref_entry_t *entry = &report->entries[report->count]; + // The hashtable stores raw pointers without owning a reference. Taking one + // here keeps every reported object alive while `__str__` runs on the others, + // since that can execute arbitrary code and drop the last reference to any + // of them. + entry->obj = Py_NewRef((PyObject *)key); + entry->refs = (Py_ssize_t)value; + report->count += 1; + return 0; +} + +static void +incoming_ref_report_clear(incoming_ref_report_t *report) +{ + for (Py_ssize_t i = 0; i < report->count; i++) { + Py_CLEAR(report->entries[i].obj); + } + report->count = 0; +} + +static PyObject * +build_close_error_message(close_error_info_t *info) +{ + incoming_ref_report_t report = {{{NULL, 0}}, 0}; + PyUnicodeWriter *writer = NULL; + + // Collect the reported objects, and with them their references, before any + // of them is formatted below. + if (_Py_hashtable_foreach(info->reported_objs, collect_incoming_ref, &report) < 0) { + goto error; + } + + writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto error; + } + + if (PyUnicodeWriter_WriteUTF8(writer, + "The region could not be closed due to:\n", -1) < 0) { + goto error; + } + + Py_ssize_t accounted = 0; + for (Py_ssize_t i = 0; i < report.count; i++) { + PyObject *obj = report.entries[i].obj; + Py_ssize_t refs = report.entries[i].refs; + accounted += refs; + + if (PyUnicodeWriter_Format(writer, + "- %zd incoming reference%s to %s '%S'\n", + refs, (refs == 1) ? "" : "s", Py_TYPE(obj)->tp_name, obj) < 0) { + goto error; + } + } + + if (accounted < info->incoming_refs) { + Py_ssize_t others = info->incoming_refs - accounted; + if (PyUnicodeWriter_Format(writer, + "- %zd reference%s to other objects\n", + others, (others == 1) ? "" : "s") < 0) { + goto error; + } + } + + incoming_ref_report_clear(&report); + return PyUnicodeWriter_Finish(writer); + +error: + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "failed to build region close error message"); + } + incoming_ref_report_clear(&report); + PyUnicodeWriter_Discard(writer); + return NULL; +} + +static int +mermaid_write_node(PyUnicodeWriter *writer, PyObject *obj) +{ + if (Region_Check(obj)) { + bool open = ((TracingRegionObject *)obj)->open; + const char *status = open ? "open" : "closed"; + return PyUnicodeWriter_Format(writer, + "n%p[\\Region
%s
rc=%zd
%p/]", + obj, status, Py_REFCNT(obj), obj); + } + if (Cown_Check(obj)) { + return PyUnicodeWriter_Format(writer, + "n%p([\"Cown
rc=%zd
%p\"])", + obj, Py_REFCNT(obj), obj); + } + return PyUnicodeWriter_Format(writer, + "n%p[\"[%s]
rc=%zd
%p\"]", + obj, Py_TYPE(obj)->tp_name, Py_REFCNT(obj), obj); +} + +static int +mermaid_write_class( + PyUnicodeWriter *writer, + PyObject *obj, + _Py_hashtable_t *problem_objs, + _Py_hashtable_t *reported_objs) +{ + if (_Py_IsImmutable(obj)) { + return PyUnicodeWriter_Format(writer, " class n%p immutable\n", obj); + } + if (_Py_hashtable_get_entry(reported_objs, obj) != NULL) { + return PyUnicodeWriter_Format(writer, " class n%p error\n", obj); + } + if (_Py_hashtable_get_entry(problem_objs, obj) != NULL) { + return PyUnicodeWriter_Format(writer, " class n%p problem\n", obj); + } + return 0; +} + +static int +mermaid_write_escaped_label(PyUnicodeWriter *writer, const char *label) +{ + for (const char *p = label; *p != '\0'; p++) { + switch (*p) { + case '|': + if (PyUnicodeWriter_WriteChar(writer, '/') < 0) { + return -1; + } + break; + case '\n': + case '\r': + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { + return -1; + } + break; + default: + if (PyUnicodeWriter_WriteChar(writer, (Py_UCS4)(unsigned char)*p) < 0) { + return -1; + } + break; + } + } + return 0; +} + +static int +mermaid_write_escaped_unicode_label(PyUnicodeWriter *writer, PyObject *label) +{ + Py_ssize_t size; + const char *utf8 = PyUnicode_AsUTF8AndSize(label, &size); + if (utf8 == NULL) { + return -1; + } + + Py_ssize_t start = 0; + for (Py_ssize_t i = 0; i < size; i++) { + switch (utf8[i]) { + case '|': + if (i > start && PyUnicodeWriter_WriteUTF8(writer, utf8 + start, i - start) < 0) { + return -1; + } + if (PyUnicodeWriter_WriteChar(writer, '/') < 0) { + return -1; + } + start = i + 1; + break; + case '\n': + case '\r': + if (i > start && PyUnicodeWriter_WriteUTF8(writer, utf8 + start, i - start) < 0) { + return -1; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { + return -1; + } + start = i + 1; + break; + default: + break; + } + } + if (size > start && PyUnicodeWriter_WriteUTF8(writer, utf8 + start, size - start) < 0) { + return -1; + } + return 0; +} + +static int +mermaid_enqueue_if_needed(mermaid_dump_state_t *state, PyObject *obj) +{ + if (_Py_IsImmutable(obj) || Cown_Check(obj)) { + return 0; + } + if (Region_Check(obj) && state->src != NULL) { + return 0; + } + if (_Py_hashtable_get_entry(state->visited, obj) != NULL) { + return 0; + } + if (_Py_hashtable_set(state->visited, obj, obj) < 0) { + PyErr_NoMemory(); + return -1; + } + return PyList_Append(state->pending, obj); +} + +static int +mermaid_visit_labeled( + PyObject *obj, + mermaid_dump_state_t *state, + const char *ascii_label, + PyObject *unicode_label) +{ + if (_Py_IsImmutable(obj) && ERROR_MERMAID_HIDE_IMMUTABLE && !Cown_Check(obj)) { + return 0; + } + + if (state->src != NULL) { + if (PyUnicodeWriter_WriteUTF8(state->writer, " ", -1) < 0) { + return -1; + } + if (mermaid_write_node(state->writer, state->src) < 0) { + return -1; + } + if (ascii_label != NULL || unicode_label != NULL) { + if (PyUnicodeWriter_WriteUTF8(state->writer, " -->|", -1) < 0) { + return -1; + } + if (ascii_label != NULL) { + if (mermaid_write_escaped_label(state->writer, ascii_label) < 0) { + return -1; + } + } + if (unicode_label != NULL && mermaid_write_escaped_unicode_label(state->writer, unicode_label) < 0) { + return -1; + } + if (PyUnicodeWriter_WriteUTF8(state->writer, "| ", -1) < 0) { + return -1; + } + } + else if (PyUnicodeWriter_WriteUTF8(state->writer, " --> ", -1) < 0) { + return -1; + } + } else if (PyUnicodeWriter_WriteUTF8(state->writer, " ", -1) < 0) { + return -1; + } + + if (mermaid_write_node(state->writer, obj) < 0) { + return -1; + } + if (PyUnicodeWriter_WriteUTF8(state->writer, "\n", -1) < 0) { + return -1; + } + if (mermaid_write_class(state->writer, obj, state->problem_objs, state->reported_objs) < 0) { + return -1; + } + + return mermaid_enqueue_if_needed(state, obj); +} + +static int +mermaid_visit(PyObject *obj, mermaid_dump_state_t *state) +{ + return mermaid_visit_labeled(obj, state, NULL, NULL); +} + +static int +mermaid_visit_dict(PyObject *obj, mermaid_dump_state_t *state) +{ + Py_ssize_t pos = 0; + PyObject *key; + PyObject *value; + + while (PyDict_Next(obj, &pos, &key, &value)) { + if (!_PyImmutability_CanViewAsImmutable(key) + && !Cown_Check(key) + && !Region_Check(key) + ) { + if (mermaid_visit_labeled(key, state, "", NULL) < 0) { + return -1; + } + } + + PyObject *label = PyUnicode_Check(key) ? key : NULL; + if (mermaid_visit_labeled(value, state, NULL, label) < 0) { + return -1; + } + } + return 0; +} + +static int +mermaid_visit_sequence(PyObject *obj, mermaid_dump_state_t *state) +{ + Py_ssize_t size = PyList_CheckExact(obj) ? PyList_GET_SIZE(obj) : PyTuple_GET_SIZE(obj); + for (Py_ssize_t i = 0; i < size; i++) { + char label[32]; + PyOS_snprintf(label, sizeof(label), "#91;%zd#93;", i); + PyObject *item = PyList_CheckExact(obj) ? PyList_GET_ITEM(obj, i) : PyTuple_GET_ITEM(obj, i); + if (mermaid_visit_labeled(item, state, label, NULL) < 0) { + return -1; + } + } + return 0; +} + +static int +mermaid_traverse(PyObject *obj, mermaid_dump_state_t *state) +{ + if (PyDict_CheckExact(obj)) { + return mermaid_visit_dict(obj, state); + } + if (PyList_CheckExact(obj) || PyTuple_CheckExact(obj)) { + return mermaid_visit_sequence(obj, state); + } + + // The trace already reports the types without tp_reachable; the graph dump + // walks the same objects and would only repeat it. + traverseproc proc = get_reachable_proc(Py_TYPE(obj), NULL); + return proc(obj, (visitproc)mermaid_visit, (void *)state); +} + +static void +mermaid_dump_state_destroy(mermaid_dump_state_t *state) +{ + if (state->writer != NULL) { + PyUnicodeWriter_Discard(state->writer); + state->writer = NULL; + } + Py_CLEAR(state->pending); + if (state->visited != NULL) { + _Py_hashtable_destroy(state->visited); + state->visited = NULL; + } +} + +static int +mermaid_dump_state_init( + mermaid_dump_state_t *state, + _Py_hashtable_t *problem_objs, + _Py_hashtable_t *reported_objs) +{ + state->writer = NULL; + state->visited = NULL; + state->pending = NULL; + state->src = NULL; + state->problem_objs = problem_objs; + state->reported_objs = reported_objs; + + state->writer = PyUnicodeWriter_Create(0); + if (state->writer == NULL) { + goto error; + } + state->visited = _Py_hashtable_new( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + if (state->visited == NULL) { + goto error; + } + state->pending = PyList_New(0); + if (state->pending == NULL) { + goto error; + } + return 0; + +error: + mermaid_dump_state_destroy(state); + return -1; +} + +static int +dump_mermaid_diagram( + PyObject *root, + _Py_hashtable_t *problem_objs, + _Py_hashtable_t *reported_objs) +{ + int res = -1; + mermaid_dump_state_t state; + PyObject *diagram = NULL; + // Owns the item currently being traversed, released at `finally`. + PyObject *item = NULL; + + // Writing a file into the working directory is too surprising to do by + // default, so the graph is only dumped when it has been asked for. The + // value of the variable is the path to write to. + const char *path = Py_GETENV(REGION_GRAPH_ENV_VAR); + if (path == NULL || *path == '\0') { + return 0; + } + + if (mermaid_dump_state_init(&state, problem_objs, reported_objs) < 0) { + return -1; + } + + if (PyUnicodeWriter_WriteUTF8(state.writer, "flowchart TD\n", -1) < 0) { + goto finally; + } + if (mermaid_visit(root, &state) < 0) { + goto finally; + } + + while (PyList_GET_SIZE(state.pending) > 0) { + Py_XSETREF(item, list_pop(state.pending)); + if (item == NULL) { + goto finally; + } + state.src = item; + SUCCEEDS(mermaid_traverse(item, &state)); + } + Py_CLEAR(item); + + diagram = PyUnicodeWriter_Finish(state.writer); + state.writer = NULL; + if (diagram == NULL) { + goto finally; + } + + const char *body = PyUnicode_AsUTF8(diagram); + if (body == NULL) { + goto finally; + } + + FILE *f = fopen(path, "w"); + if (f == NULL) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); + goto finally; + } + if (fputs( + "
\n" + "\n" + "```mermaid\n" + "%%{init: {'theme': 'neutral', 'themeVariables': { 'fontSize': '16px' }}}%%\n" + "\n", + f) < 0 + || fputs(body, f) < 0 + || fputs( + "\n" + "classDef immutable fill:#94f7ff\n" + "classDef problem fill:#ffe8d6,stroke:#f08c00,stroke-width:2px\n" + "classDef error fill:#ffe8d6,stroke:red,stroke-width:4px\n" + "```\n" + "
\n", + f) < 0) + { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); + fclose(f); + goto finally; + } + // Buffered writes can still fail here, so this result matters too. + if (fclose(f) != 0) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); + goto finally; + } + + res = 0; + +finally: + Py_XDECREF(item); + mermaid_dump_state_destroy(&state); + Py_XDECREF(diagram); + return res; +error: + goto finally; +} + +static int _move_obj(PyObject* obj, region_trace_state_t* state) { + // Check the movability of the object: + movable_status status = get_movable_status(obj); + switch (status) { + case Py_MOVABLE_YES: + break; + case Py_MOVABLE_NO: + dbg(" - %p is not movable", obj); + throw_region_error( + "Instances of type '%s' are not movable", Py_TYPE(obj)->tp_name, + state->src, obj); + return TRACE_RES_ERR; + case Py_MOVABLE_FREEZE: + // Freeze the object, this can invalidate our `external_rc`, + // we restart after this trace + dbg(" - freezing %p", obj); + if (_PyImmutability_Freeze(obj)) { + return TRACE_RES_ERR; + } + + region_trace_state_set_restart(state); + return 0; + case Py_MOVABLE_COWN: + return 0; + default: + Py_UNREACHABLE(); + } + + // References to the bridge object are allowed and counted by + // `state->bridge_rc` instead. `_trace_visit()` intercepts them, so the + // bridge must never end up in `visited` or in the LRC below. + assert(obj != state->bridge); + + // Update the LRC + Py_ssize_t lrc_change = Py_REFCNT(obj); + if (state->strong_ref) { + // -1 for the reference we just followed + lrc_change -= 1; + } + dbg(" - moving %p; LRC += %zd", obj, lrc_change); + state->external_rc += lrc_change; + + // Mark the object as visited, this stores the lrc_change for better error reporting + if (_Py_hashtable_set(state->visited, obj, (void*)lrc_change) == -1) { + PyErr_NoMemory(); + return -1; + } + + // This moves the object into the region list, if provided. + if (state->gc_list && PyObject_IS_GC(obj) && PyObject_GC_IsTracked(obj)) { + // This flag may be set if the region is constructed as part of + // a finalizer. If the flag remains set, for an object removed + // from its GC list bad things can happen. + gc_clear_collecting(_Py_AS_GC(obj)); + // Clearing the space flag makes it easy to merge this list back + // into the local GC lists + gc_set_old_space(_Py_AS_GC(obj), 0); + gc_list_move(_Py_AS_GC(obj), state->gc_list); + } + + // Bridge objects of sub-regions are moved, but shouldn't be traversed. + if (!Region_Check(obj)) { + if (PyList_Append(state->pending, obj)) { + return -1; + } + } + + return 0; +} + +static int _trace_visit_bridge_ref(PyObject* obj, region_trace_state_t* state) { + assert(Region_Check(obj)); + tree_trace_state_t *tree_state = state->tree_trace_state; + + // If the child region is closed we can move it directly + if (_PyTracingRegion_IsClosed(obj)) { + int res = _move_obj(obj, state); + // Update the region reference meta of the child, if it has one + if (res == 0) { + TracingRegionObject *child = (TracingRegionObject *)obj; + bool no_memory = false; + LOCK_REGION_REF_META(); + if (child->meta != NULL) { + _PyRegionRefMetadata *parent = + region_meta_lock_held((TracingRegionObject *)state->bridge); + if (parent == NULL) { + no_memory = true; + } + else { + _PyRegionRef_MetaSetParentLockHeld(child->meta, parent); + } + } + UNLOCK_REGION_REF_META(); + if (no_memory) { + PyErr_NoMemory(); + return TRACE_RES_ERR; + } + } + return res; + } + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(tree_state->hierarchy, (void*)obj); + if (entry == NULL) { + if (_Py_hashtable_set(tree_state->hierarchy, (void*)obj, state->bridge)) { + PyErr_NoMemory(); + return -1; + } + + void *child = (void*)state->bridge; + entry = _Py_hashtable_get_entry(tree_state->hierarchy, child); + while (entry != NULL) { + if (entry->value == obj) { + PyErr_Format( + PyExc_RuntimeError, + "the region %p can not be closed as it attempts to reference one of its parent regions %p", + (void *)obj, + entry->value); + return -1; + } + + child = entry->value; + entry = _Py_hashtable_get_entry(tree_state->hierarchy, child); + } + } else { + // We could use this branch to enforce that only a single owning + // exists for each bride. For now we allow these as long as they + // come from the same region + } + + // The child region is open, we need to traverse it first and then + // retry closing this. + if (PyList_Append(tree_state->pending, obj) < 0) { + return -1; + } + region_trace_state_set_restart(state); + + return 0; +} + +static int _trace_visit(PyObject* obj, region_trace_state_t* state) { + // References to immutable objects are allowed + if (_PyImmutability_CanViewAsImmutable(obj)) { + assert(_Py_IsImmutable(obj)); + return 0; + } + + // References to the bridge are tracked separately + if (obj == state->bridge) { + // Region objects can't have weak references + assert(state->strong_ref); + assert(get_movable_status(obj) == Py_MOVABLE_YES); + // This branch also accounts for references from the bridge object to itself. + dbg(" - Internal reference to bridge from %p; bridge_rc += 1", state->src); + state->bridge_rc += 1; + return 0; + } + + // Check if the object is already part of the region + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(state->visited, (void*)obj); + if (entry != NULL) { + assert(get_movable_status(obj) == Py_MOVABLE_YES); + // state->external_rc only counts strong references + if (state->strong_ref) { + entry->value = (void*)(((Py_ssize_t)entry->value) - 1); + dbg(" - Internal reference to %p; LRC -= 1", obj); + state->external_rc -= 1; + } + return 0; + } + + // References external regions turns them into sub-regions. These + // need to be traversed and closed separately + if (Region_Check(obj)) { + return _trace_visit_bridge_ref(obj, state); + } + + return _move_obj(obj, state); +} + + +static int _try_close_region(PyObject *region_obj, tree_trace_state_t *tree_trace_state) { + assert(Region_Check(region_obj)); + TracingRegionObject* region = (TracingRegionObject*)region_obj; + + // Finalized regions can't be closed since they're deletion would not call the + // finalizer and therefore leak the owned nodes. + if (_PyGC_FINALIZED(region_obj)) { + PyErr_Format( + PyExc_RuntimeError, + "the region %p has been finalized and cannot be closed again", + (void *)region_obj); + return TRACE_RES_ERR; + } + + // Init trace state. + region_trace_state_t state; + if (region_trace_state_init(&state, _PyObject_CAST(region), ®ion->gc_list, tree_trace_state)) { + return TRACE_RES_ERR; + } + int region_trace_res = TRACE_RES_DONE; + // Owns the item currently being traversed, released at `finally`. + PyObject *item = NULL; + + SUCCEEDS(PyList_Append(state.pending, _PyObject_CAST(region))); + + while (PyList_GET_SIZE(state.pending) > 0) { + // Find the next pending item: + Py_XSETREF(item, list_pop(state.pending)); + if (item == NULL) { + goto error; + } + + // Traverse item + state.src = item; + dbg(" - traversing %p", item); + traverseproc proc = get_reachable_proc(Py_TYPE(item), tree_trace_state->missing_reachable); + SUCCEEDS(proc(item, (visitproc)_trace_visit, (void*)&state)); + + if (PyWeakref_Check(item)) { + PyWeakReference *wref = (PyWeakReference*)item; + state.strong_ref = false; + SUCCEEDS(_trace_visit(wref->wr_object, &state)); + state.strong_ref = true; + state.has_weak_refs = true; + } + } + Py_CLEAR(item); + + if (state.restart) { + gc_list_dissolve(®ion->gc_list); + region_trace_res = TRACE_RES_RESTART; + goto finally; + } + + if (state.external_rc == 0) { + _region_close(region, state.bridge_rc, state.visited, state.has_weak_refs); + } else { + gc_list_dissolve(®ion->gc_list); + + dbg("- Failed to close region %p, there are %zd incoming references", region, state.external_rc); + close_error_info_t error_info = {0}; + if (close_error_info_init(&error_info, &state) < 0) { + goto error; + } + if (_Py_hashtable_len(state.visited) < ERROR_MERMAID_REPORT_LIMIT) { + // Borrowed error tables; dump_mermaid_diagram() does not take ownership. + if (dump_mermaid_diagram( + region_obj, + error_info.problem_objs, + error_info.reported_objs) < 0) { + // The graph is a diagnostic aid. Report why it is missing, but + // don't let that replace the region error being built here. + PyErr_FormatUnraisable( + "Exception ignored while writing the region graph"); + } + } + + PyObject *msg = build_close_error_message(&error_info); + close_error_info_destroy(&error_info); + if (msg == NULL) { + goto error; + } + PyErr_SetObject(PyExc_RuntimeError, msg); + Py_DECREF(msg); + goto error; + } + + goto finally; +error: + region_trace_res = TRACE_RES_ERR; +finally: + Py_CLEAR(item); + region_trace_state_destroy(&state); + + return region_trace_res; +} + +/* Resolves the region reference meta of every region this trace touched. + */ +static int +resolve_region_meta(_Py_hashtable_t *ht, const void *key, const void *value, + void *user_data) +{ + TracingRegionObject *region = (TracingRegionObject *)key; + if (region->meta == NULL) { + return 0; + } + // The region remains open, therefore we mark it as being local to the IP + if (region->open) { + region_meta_release(region); + return 0; + } + + // The region was closed, we resolve the WIP state + _PyRegionRef_MetaResolveWip(region->meta); + return 0; +} + +static int try_close_region_tree(PyObject *root) { + dbg("Starting region tree trace from %p", root); + + tree_trace_state_t state; + if (tree_trace_state_init(&state)) { + return -1; + } + + int tree_trace_res = TRACE_RES_DONE; + + SUCCEEDS(PyList_Append(state.pending, root)); + + while (PyList_GET_SIZE(state.pending) > 0) { + // Look at the region on top of the stack without removing it. A region + // stays queued until it is closed, so the sub-regions that its trace + // discovers end up above it and are closed first. Draining the stack + // therefore means every region in the tree is closed, which is what lets + // this function report success. + Py_ssize_t top = PyList_GET_SIZE(state.pending) - 1; + PyObject *region = PyList_GET_ITEM(state.pending, top); + assert(Region_Check(region)); + + // A closed region has nothing left to do. Regions can be queued more + // than once, this handles all safe cases. + if (_PyTracingRegion_IsClosed(region)) { + SUCCEEDS(PyList_SetSlice(state.pending, top, top + 1, NULL)); + continue; + } + + // Account for this attempt before running it. Counting afterwards would + // report a region that was closed by its last attempt as a failure, and + // would grant `PER_REGION_TRACE_LIMIT + 1` attempts. + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(state.tracing_counts, (void*)region); + if (entry == NULL) { + if (_Py_hashtable_set(state.tracing_counts, (void*)region, (void*)1) < 0) { + PyErr_NoMemory(); + goto error; + } + } else if ((Py_uintptr_t)entry->value < PER_REGION_TRACE_LIMIT) { + entry->value = (void*)(((Py_uintptr_t)entry->value) + 1); + } else { + // FIXME(regions): It would be nicer to spend the last attempt on a + // trace that reports the objects keeping the region open, like the + // `external_rc != 0` path in `_try_close_region()` does, instead of + // this bare message. The catch is that such a trace may close the + // region after all, which is why it can't simply be run here. + PyErr_Format( + PyExc_RuntimeError, + "the region %p could not be closed after %d tracing attempts", + (void *)region, + PER_REGION_TRACE_LIMIT); + goto error; + } + + dbg("- tracing region %p", region); + int res = _try_close_region(region, &state); + if (res == TRACE_RES_ERR) { + goto error; + } + // A restarted trace leaves the region open on purpose. It keeps its slot + // on the stack and is retried once the sub-regions that its trace pushed + // on top of it have been closed. + assert(res == TRACE_RES_RESTART || _PyTracingRegion_IsClosed(region)); + } + + goto finally; +error: + tree_trace_res = TRACE_RES_ERR; +finally: + // `resolve_region_meta()` never fails, so the result can be ignored. + (void)_Py_hashtable_foreach(state.tracing_counts, resolve_region_meta, NULL); + report_missing_reachable(&state); + tree_trace_state_destroy(&state); + + return tree_trace_res; +} + +// ################################################################### +// Region Object +// ################################################################### + +static PyObject * +TracingRegion_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { + TracingRegionObject *self = (TracingRegionObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + + // The region is set up here rather than in `tp_init()`, so that a region + // can never be observed in an uninitialized state. + gc_list_init(&self->gc_list); + self->meta = NULL; + // We make the region open by default, this ensures that the first close + // will handle the region type correctly. Alternatively, we could make them + // closed in the beginning, but then handle the cases specifically. + self->open = true; + + return (PyObject *)self; +} + +static int +TracingRegion_init(TracingRegionObject *self, PyObject *args, PyObject *kwargs) { + // `tp_new()` already set the region up. Re-running the initialization here + // would reset the GC list holding the contents of a closed region and drop + // the reference count that `_region_close()` subtracted from the bridge + // object, so this only validates the arguments. + if (!_PyArg_NoPositional("TracingRegion", args) + || !_PyArg_NoKeywords("TracingRegion", kwargs)) + { + return -1; + } + return 0; +} + +/* Disposes of everything a closed region owns. + * + * Closing a region establishes that no object inside it has incoming references + * from the outside; only the bridge object may have those. So once the bridge + * object dies, every member of the region is garbage too, however the references + * between them happen to be arranged. + * + * That lets the region clean up after itself instead of handing the objects back + * to the GC. + * + * This can resurrect the bridge object, so it has to run as a finalizer. + */ +static void _region_delete_contents(TracingRegionObject *self) { + assert(!self->open); + + dbg("Deleting the contents of region %p", self); + + PyGC_Head members; + PyGC_Head survivors; + gc_list_init(&members); + gc_list_init(&survivors); + + // Steal the members first. `_open_region()` will then set internal values + // but keep not invalidate `members`. + gc_list_merge(&self->gc_list, &members); + assert(gc_list_is_empty(&self->gc_list)); + _open_region(self); + + // The disposal needs a clean error state; a dealloc can happen mid-raise. + PyObject *exc = PyErr_GetRaisedException(); + + // Finalize everything before anything is released, so that no `__del__` + // observes a member that is already gone. + _PyGC_FinalizeGarbage(&members); + + // Cleaning the dict should deallocate most things. + Py_CLEAR(self->dict); + + // Deallocate remaining cyclic garbage + _PyGC_DeleteGarbage(&members, &survivors); + PyErr_SetRaisedException(exc); + + // Anything a finalizer kept alive is not owned by the region any more. + if (!gc_list_is_empty(&survivors)) { + gc_list_dissolve(&survivors); + } + // Nothing may still point at these stack allocated list heads. + assert(gc_list_is_empty(&members)); + assert(gc_list_is_empty(&survivors)); +} + +static int +TracingRegion_traverse(TracingRegionObject *self, visitproc visit, void *arg) { + Py_VISIT(self->dict); + return 0; +} + +static int +TracingRegion_clear(TracingRegionObject *self) { + _open_region(self); + Py_CLEAR(self->dict); + return 0; +} + +static void +TracingRegion_finalize(PyObject *op) { + TracingRegionObject *self = (TracingRegionObject *)op; + + if (self->open) { + assert(gc_list_is_empty(&self->gc_list)); + // An open region does not own its members. They live in the GC + // generations and the usual reference counting disposes of them. + Py_CLEAR(self->dict); + } else { + // Objects in a closed region have no incoming references besides the + // one from the bridge. We can therefore delete all objects directly + // instead of returning them to the GC. + _region_delete_contents(self); + } +} + +static void +TracingRegion_dealloc(TracingRegionObject *self) { + PyObject *op = (PyObject *)self; + + // `PyObject_CallFinalizerFromDealloc()` requires a GC type to be tracked + // while the finalizer runs, but the bridge object of a closed region may + // get untracked by an owning cown. + if (!_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_TRACK(op); + } + if (PyObject_CallFinalizerFromDealloc(op) < 0) { + // The bridge object was resurrected by the references from inside the + // region. It is deallocated again once those are gone. + return; + } + + // Make sure any objects added after/during finalization are freed + Py_CLEAR(self->dict); + + PyObject_GC_UnTrack(self); + Py_TYPE(self)->tp_free(op); +} + +static PyObject * +TracingRegion_repr(PyObject *op) { + TracingRegionObject *self = (TracingRegionObject*)op; + + // Deliberately reads `open` instead of going through the attribute access + // below, so that reporting on a region does not open it. Deliberately + // address free as well, so that error messages are reproducible. + return PyUnicode_FromFormat( + "", self->open ? "open" : "closed"); +} + +static PyObject * +TracingRegion_getattro(PyObject *op, PyObject *name) { + TracingRegionObject *self = (TracingRegionObject*)op; + _open_region(self); + + return _PyObject_GenericGetAttrWithDict(op, name, self->dict, 0); +} + +static int +TracingRegion_setattro(PyObject *op, PyObject *name, PyObject *value) { + TracingRegionObject *self = (TracingRegionObject*)op; + _open_region(self); + + // Allocate lazily because the generic helper only stores into a provided dict. + if (self->dict == NULL) { + self->dict = PyDict_New(); + if (self->dict == NULL) { + return -1; + } + } + + return _PyObject_GenericSetAttrWithDict(op, name, value, self->dict); +} + +static PyObject * +TracingRegion_get_dict(PyObject *op, void *Py_UNUSED(context)) { + TracingRegionObject *self = (TracingRegionObject*)op; + _open_region(self); + + if (self->dict == NULL) { + self->dict = PyDict_New(); + if (self->dict == NULL) { + return NULL; + } + } + return Py_NewRef(self->dict); +} + +static int +TracingRegion_set_dict(PyObject *op, PyObject *value, void *Py_UNUSED(context)) { + TracingRegionObject *self = (TracingRegionObject*)op; + _open_region(self); + + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __dict__"); + return -1; + } + if (!PyDict_Check(value)) { + PyErr_Format(PyExc_TypeError, + "__dict__ must be set to a dictionary, not a '%.200s'", + Py_TYPE(value)->tp_name); + return -1; + } + Py_XSETREF(self->dict, Py_NewRef(value)); + return 0; +} + + +/* This method traces the region and closes it, if there are no references + * pointing into the region. References to the bridge are allowed. + * + * This function requires the GIL to be held. + * + * Returns -1 if an exception was raised. 0 if the region could be closed. + */ +int _PyTracingRegion_Close(PyObject* op) { + TracingRegionObject *self = (TracingRegionObject*)op; + if (!self->open) { + return 0; + } + assert(gc_list_is_empty(&self->gc_list)); + + return try_close_region_tree(op); +} + +int _PyTracingRegion_IsClosed(PyObject* region) { + TracingRegionObject *self = (TracingRegionObject*)region; + return !self->open; +} + +/* Opens the region, so that a region reference can hand out a strong reference + * into it. Only the sub-regions of this region stay closed. + * + * This function requires the GIL to be held, and the caller to have established + * that this interpreter owns the region. + */ +void _PyTracingRegion_Open(PyObject* region) { + _open_region((TracingRegionObject*)region); +} + +_PyRegionRefMetadata *_PyTracingRegion_MetaLockHeld(PyObject* region) { + return region_meta_lock_held((TracingRegionObject*)region); +} + +void _PyTracingRegion_SetMetaCown(PyObject* region, PyObject* cown) { + TracingRegionObject *self = (TracingRegionObject*)region; + // Meta is only set if the region is closed and has region references + if (self->meta != NULL) { + _PyRegionRef_MetaSetCown(self->meta, cown); + } +} + +void _PyTracingRegion_SetMetaOwner(PyObject* region, _PyCown_ipid_t owner) { + TracingRegionObject *self = (TracingRegionObject*)region; + if (self->meta != NULL) { + _PyRegionRef_MetaSetIpid(self->meta, owner); + } +} + +static PyMethodDef TracingRegion_methods[] = { + {NULL, NULL} /* sentinel */ +}; + +static PyGetSetDef TracingRegion_getset[] = { + {"__dict__", TracingRegion_get_dict, TracingRegion_set_dict}, + {NULL} +}; + +PyTypeObject _PyTracingRegion_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "TracingRegion", + .tp_basicsize = sizeof(TracingRegionObject), + .tp_dealloc = (destructor)TracingRegion_dealloc, + .tp_repr = TracingRegion_repr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .tp_traverse = (traverseproc)TracingRegion_traverse, + .tp_clear = (inquiry)TracingRegion_clear, + .tp_getset = TracingRegion_getset, + .tp_methods = TracingRegion_methods, + .tp_getattro = TracingRegion_getattro, + .tp_setattro = TracingRegion_setattro, + .tp_init = (initproc)TracingRegion_init, + .tp_new = TracingRegion_new, + .tp_finalize = TracingRegion_finalize, + .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, +}; diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 0abf281112e7664..0ba2119ed92992a 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -6,6 +6,16 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "pycore_cown.h" // _PyCown_ThisInterpreterId() +#include "pycore_immutability.h" // _PyTracingRegion_Open() +#include "pycore_interp.h" // PyInterpreterState.immutability +#include "pycore_regionref.h" + +// FIXME(region): Reusing the same weakref easily breaks region isolation +// without a simple way out for programmers. For now we disable the optimization. +// in the future, we may create a new object every time, but make sure that +// the internal metadata is shared until this is no longer possible due to regions. +#define WEAKREF_REUSE_BASIC_REFS 0 #ifdef Py_GIL_DISABLED /* @@ -107,7 +117,476 @@ _PyWeakref_GetWeakrefCount(PyObject *obj) return count; } +// ################################################################### +// Region reference metadata +// ################################################################### + +/* See `pycore_regionref.h` for the design. + * + * Every field of a `_PyRegionRefMetadata` is guarded by `_PyWeakref_Lock`. The + * `_lock_held` helpers below expect the caller to hold it. + */ + +/* In the default build the weakref list lock and the metadata lock are the same + * global mutex, so code holding the former must not take the latter again. In + * free-threaded builds they differ. */ +#ifdef Py_GIL_DISABLED +# define LOCK_META_UNDER_WEAKREFS() LOCK_REGION_REF_META() +# define UNLOCK_META_UNDER_WEAKREFS() UNLOCK_REGION_REF_META() +#else +# define LOCK_META_UNDER_WEAKREFS() ((void)0) +# define UNLOCK_META_UNDER_WEAKREFS() ((void)0) +#endif + +static void clear_weakref_lock_held(PyWeakReference *self, PyObject **callback); + +static _PyRegionRefMetadata * +meta_new_lock_held(uint8_t kind) +{ + // Raw allocation on purpose: a node can outlive the interpreter that + // created it, when a chain reaching it is still held elsewhere. + _PyRegionRefMetadata *meta = PyMem_RawMalloc(sizeof(_PyRegionRefMetadata)); + if (meta == NULL) { + return NULL; + } + meta->rc = 1; + meta->kind = kind; + meta->region = NULL; + memset(&meta->value, 0, sizeof(meta->value)); + return meta; +} + +static void +meta_incref_lock_held(_PyRegionRefMetadata *meta) +{ + if (meta != NULL) { + assert(meta->rc > 0); + meta->rc += 1; + } +} + +static void +meta_decref_lock_held(_PyRegionRefMetadata *meta) +{ + // Releasing a node releases its parent, and a chain is as deep as the + // region tree. Walk it instead of recursing. + while (meta != NULL) { + assert(meta->rc > 0); + if (--meta->rc > 0) { + return; + } + _PyRegionRefMetadata *parent = NULL; + if (meta->kind == _Py_REGION_REF_META) { + parent = meta->value.parent; + } + PyMem_RawFree(meta); + meta = parent; + } +} + +/* Returns a terminal node owned by this interpreter, for a reference to an + * object that is in no region. New reference, NULL when out of memory. */ +static _PyRegionRefMetadata * +meta_new_local_lock_held(void) +{ + _PyRegionRefMetadata *meta = meta_new_lock_held(_Py_REGION_REF_IPID); + if (meta != NULL) { + meta->value.ipid = _PyCown_ThisInterpreterId(); + } + return meta; +} + +/* Releases whatever the node delegated to. The callers below all assign the new + * kind and value right after, so the stale union is never observed. */ +static void +meta_clear_parent_lock_held(_PyRegionRefMetadata *meta) +{ + if (meta->kind == _Py_REGION_REF_META) { + _PyRegionRefMetadata *parent = meta->value.parent; + meta->kind = _Py_REGION_REF_WIP; + meta_decref_lock_held(parent); + } +} + +static void +meta_set_parent_lock_held(_PyRegionRefMetadata *meta, _PyRegionRefMetadata *parent) +{ + assert(meta != NULL && parent != NULL); + assert(meta != parent); + + // Increfing first keeps a self-assignment from freeing the parent. + meta_incref_lock_held(parent); + meta_clear_parent_lock_held(meta); + meta->kind = _Py_REGION_REF_META; + meta->value.parent = parent; +} + +static void +meta_set_cown_lock_held(_PyRegionRefMetadata *meta, PyObject *cown) +{ + meta_clear_parent_lock_held(meta); + meta->kind = _Py_REGION_REF_COWN; + meta->value.cown = cown; +} + +static void +meta_set_ipid_lock_held(_PyRegionRefMetadata *meta, _PyCown_ipid_t ipid) +{ + meta_clear_parent_lock_held(meta); + meta->kind = _Py_REGION_REF_IPID; + meta->value.ipid = ipid; +} + +static void +set_region_ref_lock_held(PyWeakReference *self, _PyRegionRefMetadata *meta) +{ + // FIXME(regions): Why does this fail? assert(_PyRegionRef_CheckExact(self)); + if (self->region_ref == meta) { + return; + } + meta_incref_lock_held(meta); + meta_decref_lock_held(self->region_ref); + self->region_ref = meta; +} + +/* Drops a reference's metadata while its weakref list lock is held. */ +static void +clear_region_ref_lock_held(PyWeakReference *self) +{ + // FIXME(regions): Why does this fail? assert(_PyRegionRef_CheckExact(self)); + LOCK_META_UNDER_WEAKREFS(); + set_region_ref_lock_held(self, NULL); + UNLOCK_META_UNDER_WEAKREFS(); +} + +_PyRegionRefMetadata * +_PyRegionRef_NewRegionMetaLockHeld(PyObject *region) +{ + _PyRegionRefMetadata *meta = meta_new_lock_held(_Py_REGION_REF_WIP); + if (meta != NULL) { + meta->region = region; + } + return meta; +} + +void +_PyRegionRef_MetaDecref(_PyRegionRefMetadata *meta) +{ + LOCK_REGION_REF_META(); + meta_decref_lock_held(meta); + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_MetaSetParentLockHeld(_PyRegionRefMetadata *meta, + _PyRegionRefMetadata *parent) +{ + meta_set_parent_lock_held(meta, parent); +} + +void +_PyRegionRef_MetaSetCown(_PyRegionRefMetadata *meta, PyObject *cown) +{ + LOCK_REGION_REF_META(); + meta_set_cown_lock_held(meta, cown); + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_MetaSetIpid(_PyRegionRefMetadata *meta, _PyCown_ipid_t ipid) +{ + // FIXME(regions): `ipid` should always be the current interpreter. It isn't + // for a released cown, or when `PyCown_clear` runs on an interpreter that + // doesn't own the cown; once that is refactored this can assert it. + LOCK_REGION_REF_META(); + meta_set_ipid_lock_held(meta, ipid); + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_MetaRegionOpened(_PyRegionRefMetadata *meta) +{ + LOCK_REGION_REF_META(); + meta->region = NULL; + meta_set_ipid_lock_held(meta, _PyCown_ThisInterpreterId()); + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_MetaResolveWip(_PyRegionRefMetadata *meta) +{ + LOCK_REGION_REF_META(); + if (meta->kind == _Py_REGION_REF_WIP) { + meta_set_ipid_lock_held(meta, _PyCown_ThisInterpreterId()); + } + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_CloseWeakRefs(PyObject *obj, _Py_hashtable_t *keep, PyObject *region) +{ + PyWeakReference **list = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(obj); + LOCK_WEAKREFS(obj); + LOCK_META_UNDER_WEAKREFS(); + while (*list) { + PyWeakReference *ref = *list; + + // Region references remain in the list, but their region reference meta + // is repointed. + if (_PyRegionRef_CheckExact((PyObject *)ref)) { + _PyRegionRefMetadata *meta = _PyTracingRegion_MetaLockHeld(region); + if (meta == NULL) { + // Out of memory, we clear the reference and continue + clear_weakref_lock_held(ref, NULL); + continue; + } + set_region_ref_lock_held(ref, meta); + list = &ref->wr_next; + } + else if (keep != NULL && _Py_hashtable_get_entry(keep, ref)) { + list = &ref->wr_next; + } + else { + clear_weakref_lock_held(ref, NULL); + } + } + UNLOCK_META_UNDER_WEAKREFS(); + UNLOCK_WEAKREFS(obj); +} + +// ################################################################### +// Region reference access +// ################################################################### + +/* The regions a dereference has to open, innermost first. Most chains are + * shallow, so the common case stays on the stack. */ +#define REGIONREF_OPEN_STACK 8 + +typedef struct { + PyObject **items; + Py_ssize_t count; + Py_ssize_t capacity; + PyObject *stack[REGIONREF_OPEN_STACK]; +} regionref_open_list_t; + +static void +open_list_init(regionref_open_list_t *list) +{ + list->items = list->stack; + list->count = 0; + list->capacity = REGIONREF_OPEN_STACK; +} + +static void +open_list_clear(regionref_open_list_t *list) +{ + if (list->items != list->stack) { + PyMem_RawFree(list->items); + } + open_list_init(list); +} + +/* Grows with the raw allocator so this stays safe to call under the metadata + * lock, which must not run Python code. */ +static int +open_list_push(regionref_open_list_t *list, PyObject *region) +{ + if (list->count == list->capacity) { + Py_ssize_t capacity = list->capacity * 2; + PyObject **items; + if (list->items == list->stack) { + items = PyMem_RawMalloc(capacity * sizeof(PyObject *)); + if (items != NULL) { + memcpy(items, list->stack, list->count * sizeof(PyObject *)); + } + } + else { + items = PyMem_RawRealloc(list->items, capacity * sizeof(PyObject *)); + } + if (items == NULL) { + return -1; + } + list->items = items; + list->capacity = capacity; + } + list->items[list->count++] = region; + return 0; +} + +typedef enum { + REGIONREF_ALLOWED, + REGIONREF_DENIED_WIP, + REGIONREF_DENIED_IPID, + REGIONREF_DENIED_COWN, + REGIONREF_DENIED_MEMORY, +} regionref_verdict_t; + +/* Resolves the reference's metadata chain and decides whether this interpreter + * may reach the target. On success `regions` lists the regions that still have + * to be opened, innermost first; pass NULL to only ask the question. + * + * Returns 0 when access is allowed, -1 with an exception set otherwise. + */ +static int +regionref_check_access(PyWeakReference *self, regionref_open_list_t *regions, + bool quiet) +{ + const _PyCown_ipid_t this_ip = _PyCown_ThisInterpreterId(); + regionref_verdict_t verdict = REGIONREF_ALLOWED; + _PyCown_ipid_t owner = 0; + _PyCown_thread_id_t locking_thread = 0; + bool wrong_thread = false; + + // Nothing inside this section may raise or allocate through Python. + LOCK_REGION_REF_META(); + _PyRegionRefMetadata *meta = self->region_ref; + while (meta != NULL) { + if (meta->region != NULL && regions != NULL) { + if (open_list_push(regions, meta->region) < 0) { + verdict = REGIONREF_DENIED_MEMORY; + break; + } + } + if (meta->kind != _Py_REGION_REF_META) { + break; + } + meta = meta->value.parent; + } + if (verdict == REGIONREF_ALLOWED && meta != NULL) { + switch (meta->kind) { + case _Py_REGION_REF_WIP: + verdict = REGIONREF_DENIED_WIP; + break; + case _Py_REGION_REF_IPID: + owner = meta->value.ipid; + if (owner != this_ip) { + verdict = REGIONREF_DENIED_IPID; + } + break; + case _Py_REGION_REF_COWN: + owner = _PyCown_Owner(meta->value.cown); + if (owner != this_ip) { + verdict = REGIONREF_DENIED_COWN; + } + else { + locking_thread = _PyCown_LockingThread(meta->value.cown); + wrong_thread = locking_thread != _PyCown_UnsetThreadId() + && locking_thread != _PyCown_ThisThreadId(); + } + break; + default: + Py_UNREACHABLE(); + } + } + // A NULL node means the target was frozen, which makes it reachable from + // everywhere. Every live region reference has a node from birth. + UNLOCK_REGION_REF_META(); + + if (wrong_thread) { + // FIXME(regions): Thread ownership is not enforced, any thread of the + // owning interpreter may reach the data. Whether that should change is + // a question for once this has seen some use. + fprintf(stderr, + "RegionRef dereferenced from thread %llu, but the cown was " + "acquired by thread %llu\n", + (unsigned long long)_PyCown_ThisThreadId(), + (unsigned long long)locking_thread); + } + + if (verdict == REGIONREF_ALLOWED) { + return 0; + } + if (quiet) { + // Callers that only want the answer. Raising here and having them + // clear it would destroy whatever the caller already had pending; + // `repr()` in particular runs from error reporting paths. + return -1; + } + + switch (verdict) { + case REGIONREF_ALLOWED: + return 0; + case REGIONREF_DENIED_MEMORY: + PyErr_NoMemory(); + return -1; + case REGIONREF_DENIED_WIP: + PyErr_SetString( + PyExc_RuntimeError, + "the region holding this reference is currently being closed"); + return -1; + case REGIONREF_DENIED_COWN: + if (owner == _PyCown_ReleasedIpid()) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %llu attempted to dereference a region reference " + "into a released cown", + (unsigned long long)this_ip); + return -1; + } + _Py_FALLTHROUGH; + case REGIONREF_DENIED_IPID: + if (owner == _PyCown_ReleasedIpid()) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %llu attempted to dereference a region reference " + "into a region that no interpreter owns", + (unsigned long long)this_ip); + return -1; + } + PyErr_Format( + PyExc_RuntimeError, + "interpreter %llu attempted to dereference a region reference " + "into a region owned by %llu", + (unsigned long long)this_ip, (unsigned long long)owner); + return -1; + } + Py_UNREACHABLE(); +} + +/* Returns a new strong reference to the target. + * + * Returns NULL without an exception when the target simply died, and NULL with + * one set when this interpreter may not reach it. + */ +static PyObject * +regionref_get_ref(PyObject *op) +{ + PyWeakReference *self = _PyWeakref_CAST(op); + + // A dead target needs no ownership check; it is not in any region any more. + if (_Py_atomic_load_ptr(&self->wr_object) == Py_None) { + return NULL; + } + + regionref_open_list_t regions; + open_list_init(®ions); + if (regionref_check_access(self, ®ions, false) < 0) { + open_list_clear(®ions); + return NULL; + } + + // Opening runs no Python code but does move GC lists, so it happens with + // the metadata lock dropped. Parent regions first, so an open region never has a + // closed ancestor. The borrowed region pointers stay valid because the + // check above established that this interpreter owns them, and only an + // owner can deallocate a region. + for (Py_ssize_t i = regions.count - 1; i >= 0; i--) { + _PyTracingRegion_Open(regions.items[i]); + } + open_list_clear(®ions); + + PyObject *obj = _Py_atomic_load_ptr(&self->wr_object); + if (obj == Py_None) { + return NULL; + } + LOCK_WEAKREFS(obj); + PyObject *result = get_ref_lock_held(self, obj); + UNLOCK_WEAKREFS(obj); + return result; +} + static PyObject *weakref_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject *regionref_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames); static void init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) @@ -123,12 +602,16 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) else { self->callback_ipid = PyInterpreterState_GetID(PyInterpreterState_Get()); } - self->vectorcall = weakref_vectorcall; + // A region reference has to run its ownership check before handing out the + // target, so it cannot share the plain weakref fast path. + self->vectorcall = _PyRegionRef_CheckExact((PyObject *)self) + ? regionref_vectorcall : weakref_vectorcall; #ifdef Py_GIL_DISABLED self->weakrefs_lock = &WEAKREF_LIST_LOCK(ob); _PyObject_SetMaybeWeakref(ob); _PyObject_SetMaybeWeakref((PyObject *)self); #endif + self->region_ref = NULL; } // Clear the weakref and steal its callback into `callback`, if provided. @@ -157,6 +640,7 @@ clear_weakref_lock_held(PyWeakReference *self, PyObject **callback) *callback = self->wr_callback; self->wr_callback = NULL; } + clear_region_ref_lock_held(self); } // Clear the weakref and its callback @@ -190,8 +674,13 @@ void _PyWeakref_ClearRef(PyWeakReference *self) { assert(self != NULL); - assert(PyWeakref_Check(self)); + // Region references reuse this struct without being a weakref subtype. + assert(_PyWeakrefOrRegionRef_Check(self)); + // Callers here hold no lock, but `region_ref` needs one. Callers that + // already hold it use `clear_weakref_lock_held()` directly. + LOCK_REGION_REF_META(); clear_weakref_lock_held(self, NULL); + UNLOCK_REGION_REF_META(); } static void @@ -218,8 +707,12 @@ gc_clear(PyObject *op) PyWeakReference *self = _PyWeakref_CAST(op); PyObject *callback; // The world is stopped during GC in free-threaded builds. It's safe to - // call this without holding the lock. + // call this without holding the list lock. `region_ref` still needs the + // metadata lock in the default build, where each interpreter has its own + // GIL and the collector is not alone. + LOCK_REGION_REF_META(); clear_weakref_lock_held(self, &callback); + UNLOCK_REGION_REF_META(); Py_XDECREF(callback); return 0; } @@ -381,7 +874,7 @@ static PyWeakReference * try_reuse_basic_ref(PyWeakReference *list, PyTypeObject *type, PyObject *callback) { - if (callback != NULL) { + if (!WEAKREF_REUSE_BASIC_REFS || callback != NULL) { return NULL; } @@ -483,11 +976,16 @@ _PyWeakref_OnObjectFreeze(PyObject *object) return; } LOCK_WEAKREFS(object); + LOCK_META_UNDER_WEAKREFS(); PyWeakReference *current = *list; while (current != NULL) { + // A frozen object is reachable from every interpreter, so a region + // reference to it no longer needs an ownership check. + set_region_ref_lock_held(current, NULL); immutable_make_weakref_safe(current); current = current->wr_next; } + UNLOCK_META_UNDER_WEAKREFS(); UNLOCK_WEAKREFS(object); } @@ -606,6 +1104,9 @@ _PyWeakref_RefType = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_BASETYPE, .tp_traverse = gc_traverse, + // tp_reachable explicitly doesn't visit the weak reference to reflect the + // actual RC of referenced objects. Changes to this will require adjustments + // in freezing and region traversal code. .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, .tp_clear = gc_clear, .tp_richcompare = weakref_richcompare, @@ -617,6 +1118,235 @@ _PyWeakref_RefType = { .tp_free = PyObject_GC_Del, }; +// ################################################################### +// Region reference type +// ################################################################### + +/* A region reference is a weak reference that survives its target's region + * being closed. Instead of keeping the region open, every dereference asks + * whether the target may be reached and opens the region tree if it + * may. + * + * This type is deliberately not a subtype of `_PyWeakref_RefType`, which is + * what keeps `PyWeakref_Check()` false for it. + */ + +static PyObject * +regionref_vectorcall(PyObject *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + if (!_PyArg_NoKwnames("RegionRef", kwnames)) { + return NULL; + } + if (!_PyArg_CheckPositional("RegionRef", PyVectorcall_NARGS(nargsf), 0, 0)) { + return NULL; + } + PyObject *obj = regionref_get_ref(self); + if (obj == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; + } + return obj; +} + +static Py_hash_t +regionref_hash(PyObject *op) +{ + PyWeakReference *self = _PyWeakref_CAST(op); + // Checked before the cache is consulted: a cached hash would otherwise be + // a standing answer about an object this interpreter may no longer touch. + if (regionref_check_access(self, NULL, false) < 0) { + return -1; + } + Py_hash_t hash = _Py_atomic_load_ssize_relaxed(&self->hash); + if (hash != -1) { + return hash; + } + PyObject *obj = regionref_get_ref(op); + if (obj == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, "weak object has gone away"); + } + return -1; + } + hash = PyObject_Hash(obj); + Py_DECREF(obj); + _Py_atomic_store_ssize_relaxed(&self->hash, hash); + return hash; +} + +static PyObject * +regionref_repr(PyObject *self) +{ + PyWeakReference *ref = _PyWeakref_CAST(self); + PyObject *obj = _Py_atomic_load_ptr(&ref->wr_object); + if (obj == Py_None) { + return PyUnicode_FromFormat(""); + } + + // Deliberately only asks the question instead of going through + // `regionref_get_ref()`, so that printing a reference never opens a + // region, matching `TracingRegion`'s repr. + if (regionref_check_access(ref, NULL, true) < 0) { + return PyUnicode_FromFormat(""); + } + + // This needs a lock, since the object may be in the middle of finalizing when this + // is being called. + LOCK_WEAKREFS(obj); + PyObject *target = get_ref_lock_held(ref, obj); + UNLOCK_WEAKREFS(obj); + if (target == NULL) { + return PyUnicode_FromFormat(""); + } + PyObject *repr = PyUnicode_FromFormat( + "", self, target, target); + Py_DECREF(target); + return repr; +} + +/* Region references only support equality, and compare by target like weak + * references do. A reference whose target is gone or out of reach falls back to + * identity, since there is nothing to compare. */ +static PyObject * +regionref_richcompare(PyObject *self, PyObject *other, int op) +{ + if ((op != Py_EQ && op != Py_NE) + || !_PyRegionRef_CheckExact(self) + || !_PyRegionRef_CheckExact(other)) + { + Py_RETURN_NOTIMPLEMENTED; + } + + // An unreachable target compares by identity, like a dead one. The check + // runs quietly so that a denial never disturbs the caller's error state. + PyObject *obj = NULL; + PyObject *other_obj = NULL; + if (regionref_check_access(_PyWeakref_CAST(self), NULL, true) == 0) { + obj = regionref_get_ref(self); + } + if (regionref_check_access(_PyWeakref_CAST(other), NULL, true) == 0) { + other_obj = regionref_get_ref(other); + } + if (PyErr_Occurred()) { + Py_XDECREF(obj); + Py_XDECREF(other_obj); + return NULL; + } + + if (obj == NULL || other_obj == NULL) { + Py_XDECREF(obj); + Py_XDECREF(other_obj); + int res = (self == other); + if (op == Py_NE) { + res = !res; + } + return PyBool_FromLong(res); + } + + PyObject *res = PyObject_RichCompare(obj, other_obj, op); + Py_DECREF(obj); + Py_DECREF(other_obj); + return res; +} + +static PyObject * +regionref___new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + // FIXME(regions): Region references do not support callbacks yet. Adding + // them means deciding which interpreter runs the callback and how. + if (!_PyArg_NoKeywords("RegionRef", kwargs)) { + return NULL; + } + PyObject *ob; + if (!PyArg_UnpackTuple(args, "__new__", 1, 1, &ob)) { + return NULL; + } + + PyWeakReference *ref = get_or_create_weakref(type, ob, NULL); + if (ref == NULL) { + return NULL; + } + + // FIXME(region): Freezing a region reference needs special handling like weak + // references. We also need to handle a case, where a freeze would propagate into + // a closed region. The solution is probably a pre-freeze hook that calls freeze + // on the target object. + if (_PyImmutability_SetFreezable( + (PyObject *)ref, _Py_FREEZABLE_NO) < 0) { + Py_DECREF(ref); + return NULL; + } + + // An immutable target is reachable from everywhere, no meta is set. + if (_Py_IsImmutable(ob)) { + return (PyObject *)ref; + } + + // Until a close re-homes it, the target is local to this interpreter. + LOCK_REGION_REF_META(); + _PyRegionRefMetadata *meta = meta_new_local_lock_held(); + if (meta != NULL) { + set_region_ref_lock_held(ref, meta); + meta_decref_lock_held(meta); + } + UNLOCK_REGION_REF_META(); + if (meta == NULL) { + Py_DECREF(ref); + return PyErr_NoMemory(); + } + + return (PyObject *)ref; +} + +static int +regionref___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (!_PyArg_NoKeywords("RegionRef", kwargs)) { + return -1; + } + PyObject *tmp; + return PyArg_UnpackTuple(args, "__init__", 1, 1, &tmp) ? 0 : -1; +} + +PyDoc_STRVAR(regionref_doc, +"RegionRef(object)\n\ +--\n\ +\n\ +A weak reference into a region that does not keep the region open.\n\ +Calling it returns the referenced object if th object may be reached,\n\ +or raises a RuntimeError otherwise. Returns None once the referenced \n\ +object is gone."); + +PyTypeObject +_PyRegionref_RefType = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "immutable.RegionRef", + .tp_basicsize = sizeof(PyWeakReference), + .tp_dealloc = weakref_dealloc, + .tp_vectorcall_offset = offsetof(PyWeakReference, vectorcall), + .tp_call = PyVectorcall_Call, + .tp_repr = regionref_repr, + .tp_hash = regionref_hash, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_HAVE_VECTORCALL, + .tp_doc = regionref_doc, + .tp_traverse = gc_traverse, + // tp_reachable explicitly doesn't visit the weak reference to reflect the + // actual RC of referenced objects. Changes to this will require adjustments + // in freezing and region traversal code. + .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, + .tp_clear = gc_clear, + .tp_richcompare = regionref_richcompare, + .tp_methods = weakref_methods, + .tp_init = regionref___init__, + .tp_alloc = PyType_GenericAlloc, + .tp_new = regionref___new__, + .tp_free = PyObject_GC_Del, +}; + static bool proxy_check_ref(PyObject *obj) @@ -1251,6 +1981,12 @@ _PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type) void _PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj) +{ + _PyWeakref_ClearWeakRefsExcept(obj, NULL); +} + +void +_PyWeakref_ClearWeakRefsExcept(PyObject *obj, _Py_hashtable_t *keep) { /* Modeled after GET_WEAKREFS_LISTPTR(). @@ -1259,7 +1995,11 @@ _PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj) PyWeakReference **list = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(obj); LOCK_WEAKREFS(obj); while (*list) { - _PyWeakref_ClearRef(*list); + if (keep != NULL && _Py_hashtable_get_entry(keep, *list)) { + list = &((*list)->wr_next); + } else { + clear_weakref_lock_held(*list, NULL); + } } UNLOCK_WEAKREFS(obj); } diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 3702e4e99987183..c19b7efdd181537 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -134,6 +134,7 @@ + @@ -161,6 +162,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 0b968eba5b977bf..0a33235d9dc855c 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -106,6 +106,9 @@ Source Files + + Source Files + Source Files @@ -478,6 +481,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index d6ce53bbea28245..32d5877122e4948 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -238,6 +238,7 @@ + @@ -532,6 +533,7 @@ + @@ -559,6 +561,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index d5351a82741a0fe..0106e8290c20fa5 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -618,6 +618,9 @@ Include\internal + + Include\internal + Include\internal @@ -697,6 +700,8 @@ Include\internal + Include\internal + Include\internal @@ -1207,6 +1212,9 @@ Objects + + Objects + Objects @@ -1273,6 +1281,9 @@ Objects + + Objects + Objects diff --git a/Python/gc.c b/Python/gc.c index 91f50486cda01ce..4bd73e4143db9a5 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -968,7 +968,7 @@ handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old) * Since the callback is never needed and may be unsafe in this * case, wr is simply left in the unreachable set. Note that * clear_weakrefs() will ensure its callback will not trigger - * inside delete_garbage(). + * inside _PyGC_DeleteGarbage(). * * OTOH, if wr isn't part of CT, we should invoke the callback: the * weakref outlived the trash. Note that since wr isn't CT in this @@ -1061,10 +1061,11 @@ clear_weakrefs(PyGC_Head *unreachable) PyObject *op = FROM_GC(gc); next = GC_NEXT(gc); - if (PyWeakref_Check(op)) { + if (_PyWeakrefOrRegionRef_Check(op)) { /* A weakref inside the unreachable set is always cleared. See * the comments above handle_weakref_callbacks() for why these - * must be cleared. + * must be cleared. Region references reuse the same struct + * without being a weakref subtype, and need it just as much. */ _PyWeakref_ClearRef((PyWeakReference *)op); } @@ -1136,9 +1137,10 @@ handle_legacy_finalizers(PyThreadState *tstate, * Note that this may remove some (or even all) of the objects from the * list, due to refcounts falling to 0. */ -static void -finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable) +void +_PyGC_FinalizeGarbage(PyGC_Head *collectable) { + PyThreadState *tstate = _PyThreadState_GET(); destructor finalize; PyGC_Head seen; @@ -1173,10 +1175,12 @@ finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable) * tricky business as the lists can be changing and we don't know which * objects may be freed. It is possible I screwed something up here. */ -static void -delete_garbage(PyThreadState *tstate, GCState *gcstate, - PyGC_Head *collectable, PyGC_Head *old) +void +_PyGC_DeleteGarbage(PyGC_Head *collectable, PyGC_Head *old) { + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; + assert(!_PyErr_Occurred(tstate)); while (!gc_list_is_empty(collectable)) { @@ -1796,7 +1800,7 @@ gc_collect_region(PyThreadState *tstate, validate_list(&unreachable, collecting_set_unreachable_clear); /* Call tp_finalize on objects which have one. */ - finalize_garbage(tstate, &unreachable); + _PyGC_FinalizeGarbage(&unreachable); /* Handle any objects that may have resurrected after the call * to 'finalize_garbage' and continue the collection with the * objects that are still unreachable */ @@ -1814,7 +1818,7 @@ gc_collect_region(PyThreadState *tstate, * in finalizers to be freed. */ stats->collected += gc_list_size(&final_unreachable); - delete_garbage(tstate, gcstate, &final_unreachable, to); + _PyGC_DeleteGarbage(&final_unreachable, to); /* Collect statistics on uncollectable objects found and print * debugging information. */ diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index b08d74b1c2f8a3a..1f97f94c297e830 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1590,8 +1590,10 @@ clear_weakrefs(struct collection_state *state) { PyObject *op; WORKSTACK_FOR_EACH(&state->unreachable, op) { - if (PyWeakref_Check(op)) { - // Clear weakrefs that are themselves unreachable. + if (_PyWeakrefOrRegionRef_Check(op)) { + // Clear weakrefs that are themselves unreachable. Region + // references reuse the same struct without being a weakref + // subtype, and need clearing just as much. _PyWeakref_ClearRef((PyWeakReference *)op); } diff --git a/Python/immutability.c b/Python/immutability.c index c4feb45d0511c7b..9ca0901159beada 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -1828,6 +1828,14 @@ int _PyImmutability_CanViewAsImmutable(PyObject *obj) } _Py_hashtable_destroy(state.visited); + + // We can't call the destructor directly as we didn't newref the objects + // on push. Breaking out of the loop above leaves the remaining objects + // on the worklist, so drain it here. This is a slow path if there are + // still objects in the stack, so there is no need to optimize it. + while (PyList_Size(state.worklist) > 0) { + pop(state.worklist); + } Py_DECREF(state.worklist); if (result < 0) { diff --git a/region-error-plan.md b/region-error-plan.md new file mode 100644 index 000000000000000..01b1e1874beed7d --- /dev/null +++ b/region-error-plan.md @@ -0,0 +1,158 @@ +## Plan: restore region close diagnostics + +### Current state + +The tree-region rewrite moved the close algorithm into `try_close_region_tree()` and `_try_close_region()` in `Objects/tracingregionobject.c`. The good news is that the important accounting still exists locally: + +- `region_trace_state_t.visited` still maps each moved object to its local refcount delta. +- `region_trace_state_t.external_rc` still carries the total outstanding incoming references for the region being traced. +- `region_trace_state_t.src` still identifies the source object during traversal, which is enough to rebuild graph edges. +- `_try_close_region()` still has the exact failure point where diagnostics should be produced, after dissolving the tentative GC list and after ignoring restart traces. + +The regression is mostly that `_try_close_region()` now formats only: + +```text +Failed to close region %p, there are %zd incoming references +``` + +and then destroys `state.visited`, so the object-level detail and Mermaid graph are lost. The old implementation had these pieces before the rewrite: + +- `error_ref_filter` / `_filter_visited()` to select the first `ERROR_OBJECT_REPORT_COUNT` objects with positive incoming references. +- `build_close_error_message()` to emit: + - `The region could not be closed due to:` + - `- N incoming reference(s) to 'obj'` + - `- N reference(s) to other objects` +- `mermaid_builder_t`, `mermaid_visit()`, and `dump_mermaid_diagram()` to write `region-graph.md` with red-highlighted leaking objects and cyan immutable objects. + +### Desired behavior + +When closing a region tree fails because a particular open region has lingering references into it, the exception should again identify the problematic objects instead of only reporting a total count. For small graphs, the failed close should also regenerate `region-graph.md` so the reference path can be inspected visually. + +The diagnostics should be scoped to the region that actually failed during `try_close_region_tree()`, not to the whole tree unless a later failure aggregation is explicitly added. That preserves the current close algorithm: child regions are closed first; the parent is retried; whichever region still has external refs reports its own graph. + +### Implementation steps + +1. Reintroduce a diagnostic result type. + + Add a small struct near the trace state types, for example: + + ```c + typedef struct { + _Py_hashtable_t *obj_table; + Py_ssize_t incoming_refs; + } close_error_info_t; + ``` + + Keep it separate from `region_trace_state_t` so the tracing state can remain reusable and the caller owns the filtered error table lifetime. + +2. Re-add the filtering helpers, adjusted for bridge semantics. + + Restore the old `error_ref_filter` idea, but make it explicit that the bridge object has one expected external owning reference. In the old code this was handled by subtracting one from the root region object; in the new code the bridge is `state.bridge`. + + Rules for `_filter_visited()`: + + - Start with the stored ref delta from `state.visited`. + - If `key == state.bridge`, subtract the expected owning reference. + - Keep only entries with `refs > 0`. + - Cap the table at `ERROR_OBJECT_REPORT_COUNT` entries. + - Treat `_Py_hashtable_foreach()` return `1` as intentional early stop, not an error. + + This preserves the old message shape while matching the current close model, where references to the bridge from inside the region are tracked separately as `bridge_rc` and should not be reported as external leaks. + +3. Build diagnostics inside `_try_close_region()` before destroying `state`. + + In the `state.external_rc != 0` failure branch, after `gc_list_dissolve(®ion->gc_list)` and after the `state.restart` check: + + - Allocate the filtered `close_error_info_t.obj_table` from `state.visited`. + - Store `close_error_info_t.incoming_refs = state.external_rc`. + - Build the Python exception with the restored `build_close_error_message()`. + - Fall back to the existing summary string only if message construction fails without a more specific exception. + - Destroy the filtered table on all exits. + + Important: do not build the nice error on restart traces. Restart traces are intentionally incomplete because freezing or open child-region discovery invalidated the current accounting. + +4. Restore `build_close_error_message()`. + + Port the old `incoming_ref_report`, `_report_incoming_ref()`, and `build_close_error_message()` almost directly. The main adjustment is replacing `trace_info_t` with `close_error_info_t` and making the expected-reference subtraction happen during filtering, not during final summarization. + + The summary calculation should therefore be: + + ```c + Py_ssize_t problem_refs = error_info->incoming_refs; + ``` + + not `incoming_refs - 1`, because the bridge's expected reference has already been removed from the filtered object counts and should also be excluded from `external_rc` if needed. If `external_rc` still includes the expected bridge reference for the region currently being closed, subtract it once at diagnostic collection time and document that invariant next to the code. + + Cheap check: the existing `test_release_error` expectations in `Lib/test/test_freeze/test_tracing_region.py` should pass with the old exact message lines. + +5. Re-add Mermaid generation as a read-only diagnostic trace. + + Restore `mermaid_builder_t` and `mermaid_visit()`, but adapt it to `region_trace_state_t`: + + - Add `mermaid_builder_t *mermaid;` to `region_trace_state_t`, initialized to `NULL` in `region_trace_state_reset()`. + - At the start of `_trace_visit()`, call `mermaid_visit(obj, state)` when `state->mermaid != NULL`. + - In `mermaid_visit()`, keep the old node format: pointer, refcount, and type name. + - Preserve the old special node shapes for ownership objects: + - Regions use Mermaid's subroutine shape: `id[[Region 0x...]]`. + - Cowns use Mermaid's stadium shape: `id([Cown 0x...])`. + - Continue hiding immutable nodes behind `ERROR_MERMAID_HIDE_IMMUTABLE`. + - Highlight objects present in the filtered error table with `:::error`. + + For the diagnostic trace, initialize `region_trace_state_t` with `gc_list == NULL` so no objects are moved. Use the same `tree_trace_state_t` shape only if required by `_trace_visit()` for region references; otherwise, split a read-only Mermaid visitor path from closing behavior so dumping the graph cannot enqueue or close subregions. + +6. Decide how Mermaid handles sub-regions. + + The tree rewrite adds a case the old graph did not have: references to region bridge objects can represent nested ownership rather than ordinary objects. + + Recommended first version: + + - Show closed sub-region bridge objects as region-shaped boundary nodes and do not traverse into them, matching `_move_obj()`'s current `if (!Region_Check(obj))` behavior. + - Treat open sub-regions as boundary nodes in the graph and label them as `[TracingRegion open]` or `[TracingRegion closed]` if that can be done without allocating risky strings. + - Do not let Mermaid dumping trigger `_enqueue_region_for_closing()` or `region_trace_state_set_restart()`. + - For now, dump only the graph for the single region that failed. Do not attempt to show the whole region tree yet. + + This keeps the diagnostic graph side-effect-free and aligned with the current failure point. A later enhancement can add dashed edges from parent to child region graphs if whole-tree visualization becomes useful. + +7. Write `region-graph.md` only when the graph is small. + + Reuse the old limit: + + ```c + if (_Py_hashtable_len(state.visited) < ERROR_MERMAID_REPORT_LIMIT) { + dump_mermaid_diagram(region_obj, error_info.obj_table); + } + ``` + + Keep the graph dump strictly best-effort: failure to open `region-graph.md` must not replace the close error. Actual Python exceptions from building the diagram should either be cleared and ignored, or avoided by making the dump path best-effort all the way through. For diagnostics, losing the graph is less important than preserving the close failure message. + +8. Add focused tests. + + Update or add tests in `Lib/test/test_freeze/test_tracing_region.py`: + + - Keep the existing simple leak test for exact message shape. + - Add a capped-output test with more than `ERROR_OBJECT_REPORT_COUNT` leaked objects and an `other objects` summary. + - Add a tree-region case where a child region fails to close and the error names an object inside the child, not just the parent total. + - Add a tree-region case where the child closes successfully but the parent fails due to a reference into the parent. + - For Mermaid, either assert that `region-graph.md` exists and contains `flowchart TD` plus `:::error`, or add a small C-visible/private Python hook if file-system assertions are too brittle. + + Also fix the duplicate Python test method name currently present in `TestTraceRefs`; the second `test_release_error` overrides the first. + +9. Validation commands. + + After implementation, run the narrow test file first: + + ```sh + ./python.exe -m test test_freeze.test_tracing_region + ``` + + Then run a build if C changes were made: + + ```sh + make -j + ``` + +### Decisions for this pass + +- `region-graph.md` should show only the single failing region for now. +- Graph dumping is strictly best-effort; diagnostic file failures should not mask the ownership violation. +- Structured exception attributes like `source` and `target` are a follow-up, not part of this restoration pass.