diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 55f8e046e35e14b..7ed70505a21a4bc 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -2,6 +2,18 @@ # error "this header file must not be included directly" #endif +/* A region reference is a weak reference that survives its target's region + * being closed. Instead of keeping the region open it checks on every + * dereference whether this interpreter may reach the target, and opens the + * region tree if it may. The metadata carrying that information lives in + * `pycore_regionref.h`; it is opaque here. + */ +PyAPI_DATA(PyTypeObject) _PyRegionref_RefType; + +#define _PyRegionRef_CheckExact(op) Py_IS_TYPE((op), &_PyRegionref_RefType) + +struct _PyRegionRefMetadata; + /* PyWeakReference is the base struct for the Python ReferenceType, ProxyType, * and CallableProxyType. */ @@ -43,12 +55,23 @@ struct _PyWeakReference { */ PyMutex *weakrefs_lock; #endif + + /* The ownership domain of `wr_object`, or NULL if this object doesn't have an ownership + * domain. This can happen if this is a normal weakref or if the object is immutable. + */ + struct _PyRegionRefMetadata *region_ref; }; PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); +/* Region references reuse this struct but are deliberately not a subtype of + * `_PyWeakref_RefType`, so that `PyWeakref_Check()` stays false for them and + * the region close trace does not follow them. */ +#define _PyWeakrefOrRegionRef_Check(op) \ + (PyWeakref_Check(op) || _PyRegionRef_CheckExact(op)) + #define _PyWeakref_CAST(op) \ - (assert(PyWeakref_Check(op)), _Py_CAST(PyWeakReference*, (op))) + (assert(_PyWeakrefOrRegionRef_Check(op)), _Py_CAST(PyWeakReference*, (op))) // Test if a weak reference is dead. PyAPI_FUNC(int) PyWeakref_IsDead(PyObject *ref); diff --git a/Include/internal/pycore_cown.h b/Include/internal/pycore_cown.h index 0345690cc015ab6..60fe3058df61bd6 100644 --- a/Include/internal/pycore_cown.h +++ b/Include/internal/pycore_cown.h @@ -22,6 +22,16 @@ typedef uint64_t _PyCown_thread_id_t; PyAPI_FUNC(_PyCown_ipid_t) _PyCown_ThisInterpreterId(void); PyAPI_FUNC(_PyCown_thread_id_t) _PyCown_ThisThreadId(void); +/* The interpreter currently owning the cown, or `_PyCown_ReleasedIpid()` when + * no interpreter does. Safe to call from any interpreter. */ +PyAPI_FUNC(_PyCown_ipid_t) _PyCown_Owner(PyObject *cown); +PyAPI_FUNC(_PyCown_ipid_t) _PyCown_ReleasedIpid(void); + +/* The thread that acquired the cown, or `_PyCown_UnsetThreadId()` when it was + * acquired without the GIL. Not enforced, only reported. */ +PyAPI_FUNC(_PyCown_thread_id_t) _PyCown_LockingThread(PyObject *cown); +PyAPI_FUNC(_PyCown_thread_id_t) _PyCown_UnsetThreadId(void); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_immutability.h b/Include/internal/pycore_immutability.h index 32b49580c56d775..d883e44c69c7268 100644 --- a/Include/internal/pycore_immutability.h +++ b/Include/internal/pycore_immutability.h @@ -8,9 +8,32 @@ extern "C" { # error "Py_BUILD_CORE must be defined to include this header" #endif +struct _PyRegionRefMetadata; + PyAPI_DATA(PyTypeObject) _PyTracingRegion_Type; PyAPI_FUNC(int) _PyTracingRegion_Close(PyObject* region); PyAPI_FUNC(int) _PyTracingRegion_IsClosed(PyObject* region); +PyAPI_FUNC(void) _PyTracingRegion_Open(PyObject* region); + +/* Returns the region's metadata node, allocating it if this is the first + * region reference the current close has found. Borrowed, and only valid while + * the region stays closed. The caller must hold `_PyWeakref_Lock`. */ +PyAPI_FUNC(struct _PyRegionRefMetadata*) + _PyTracingRegion_MetaLockHeld(PyObject* region); + +/* Hands a closed region's node to `cown`, used when a cown takes ownership of + * the region. Does nothing for an open region. */ +PyAPI_FUNC(void) _PyTracingRegion_SetMetaCown(PyObject* region, PyObject* cown); + +/* Records who owns a closed region, used when it leaves the cown that owned it. + * The owner is NOT necessarily the calling interpreter: a cown is immutable and + * may be deallocated by anyone holding a reference, including an interpreter + * that never owned it. Pass `_PyCown_ReleasedIpid()` when nobody owns it. + * Does nothing for an open region. */ +// FIXME: The deallocation will be fixed in a follow-up, then we can remove the +// owner argument and assert that it's always local. +PyAPI_FUNC(void) _PyTracingRegion_SetMetaOwner( + PyObject* region, uint64_t owner); struct _Py_immutability_state { int late_init_done; diff --git a/Include/internal/pycore_regionref.h b/Include/internal/pycore_regionref.h new file mode 100644 index 000000000000000..acdff81d6df39e9 --- /dev/null +++ b/Include/internal/pycore_regionref.h @@ -0,0 +1,86 @@ +#ifndef Py_INTERNAL_REGIONREF_H +#define Py_INTERNAL_REGIONREF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +#include "pycore_cown.h" // _PyCown_ipid_t +#include "pycore_hashtable.h" // _Py_hashtable_t + +/* Every `RegionRef` points at a `_PyRegionRefMetadata` node, and those nodes + * form a tree that mirrors the region hierarchy: a nested region's node + * delegates to the node of its parent, and the outermost node names the owner, + * either an interpreter or the cown holding the region. A cown node looks the + * owner up on the cown, which is what makes acquiring and releasing free; + * moving a closed region between owners restamps a single node. + * + * A reference to an object that is in no region at all owns its own node. + */ + +typedef enum { + /* A close is in progress. The referenced data is unavailable but should be + * soon; dereferencing fails and the caller may retry. Only ever reachable + * through a child's `parent`, never directly from a reference. */ + _Py_REGION_REF_WIP, + /* Delegates to `value.parent`, the node of the enclosing region. */ + _Py_REGION_REF_META, + /* Terminal. The region is held by `value.cown`, on which the owner is + * looked up dynamically. */ + _Py_REGION_REF_COWN, + /* Terminal, owned by one interpreter. */ + _Py_REGION_REF_IPID, +} _PyRegionRefKind; + +typedef struct _PyRegionRefMetadata { + uint32_t rc; + /* A `_PyRegionRefKind`. */ + uint8_t kind; + /* Borrowed. Set only on region nodes, and only while that region is + * closed. Names the region a dereference has to open on its way down. + * + * Borrowing is safe because the pointer is only followed after the terminal + * check established that this interpreter owns the region, and a region can + * only be deallocated by its owner. */ + PyObject *region; + union { + struct _PyRegionRefMetadata *parent; /* META */ + PyObject *cown; /* COWN, borrowed */ + _PyCown_ipid_t ipid; /* IPID */ + } value; +} _PyRegionRefMetadata; + +/* Creates the node of a region that is being closed. Returns a new reference. */ +extern _PyRegionRefMetadata *_PyRegionRef_NewRegionMetaLockHeld(PyObject *region); + +extern void _PyRegionRef_MetaDecref(_PyRegionRefMetadata *meta); + +// Ownership transitions. +extern void _PyRegionRef_MetaSetParentLockHeld(_PyRegionRefMetadata *meta, + _PyRegionRefMetadata *parent); +/* Hands the node to `cown`, which is borrowed. The owner is from then on + * whoever holds the cown. */ +extern void _PyRegionRef_MetaSetCown(_PyRegionRefMetadata *meta, PyObject *cown); +/* Stamps an explicit owner, which need not be the current interpreter and may + * be `_PyCown_ReleasedIpid()` to mean nobody owns the region. */ +extern void _PyRegionRef_MetaSetIpid(_PyRegionRefMetadata *meta, + _PyCown_ipid_t ipid); +extern void _PyRegionRef_MetaRegionOpened(_PyRegionRefMetadata *meta); +extern void _PyRegionRef_MetaResolveWip(_PyRegionRefMetadata *meta); + +/* Re-homes every `RegionRef` pointing at `obj` onto `region`'s node, allocating + * that node if this is the first reference the close has found. Every other + * weak reference to `obj` is cleared unless it is listed in `keep`. + * + * This is the region close hook; it replaces `_PyWeakref_ClearWeakRefsExcept()` + * for objects that are being closed into a region. */ +extern void _PyRegionRef_CloseWeakRefs(PyObject *obj, _Py_hashtable_t *keep, + PyObject *region); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_REGIONREF_H */ diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index f9bc2e000c68c34..cced88c3fdcce33 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -13,6 +13,15 @@ extern "C" { #include "pycore_object.h" // _Py_REF_IS_MERGED() #include "pycore_pyatomic_ft_wrappers.h" +/* Guards weakrefs to immutable objects, and all region reference metadata. + * Declared for both builds because `_PyRegionRefMetadata` uses it either way, + * while the weakref lists themselves are striped in free-threaded builds. */ +extern PyMutex _PyWeakref_Lock; + +#define LOCK_REGION_REF_META() \ + PyMutex_LockFlags(&_PyWeakref_Lock, _Py_LOCK_DONT_DETACH) +#define UNLOCK_REGION_REF_META() PyMutex_Unlock(&_PyWeakref_Lock) + #ifdef Py_GIL_DISABLED #define WEAKREF_LIST_LOCK(obj) \ @@ -37,9 +46,6 @@ extern "C" { #else -// Lock used for weakrefs to immutable objects -extern PyMutex _PyWeakref_Lock; - #define LOCK_WEAKREFS(obj) PyMutex_LockFlags(&_PyWeakref_Lock, _Py_LOCK_DONT_DETACH) #define UNLOCK_WEAKREFS(obj) PyMutex_Unlock(&_PyWeakref_Lock) @@ -108,7 +114,7 @@ static inline PyObject* get_ref_lock_held(PyWeakReference *ref, PyObject *obj) static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) { - assert(PyWeakref_Check(ref_obj)); + assert(_PyWeakrefOrRegionRef_Check(ref_obj)); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); PyObject *obj = _Py_atomic_load_ptr(&ref->wr_object); @@ -125,7 +131,7 @@ static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) { - assert(PyWeakref_Check(ref_obj)); + assert(_PyWeakrefOrRegionRef_Check(ref_obj)); int ret = 0; PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); diff --git a/Lib/immutable.py b/Lib/immutable.py index 167273bc31cdbd6..30ec5b32b2532d0 100644 --- a/Lib/immutable.py +++ b/Lib/immutable.py @@ -23,6 +23,7 @@ SharedField = _c.SharedField TracingRegion = _c.TracingRegion Cown = _c.Cown +RegionRef = _c.RegionRef # FIXME(immutable): For the longest time we used the name `isfrozen` # without the underscore. This keeps the function name for now, but @@ -141,6 +142,9 @@ def __enter__(self): "FREEZABLE_PROXY", "InterpreterLocal", "SharedField", + "TracingRegion", + "Cown", + "RegionRef", "freezable", "unfreezable", "explicitlyFreezable", diff --git a/Lib/test/test_freeze/test_tracing_region.py b/Lib/test/test_freeze/test_tracing_region.py index 946d74239ed9b72..ae1701519cf921c 100644 --- a/Lib/test/test_freeze/test_tracing_region.py +++ b/Lib/test/test_freeze/test_tracing_region.py @@ -5,7 +5,8 @@ import weakref from immutable import freeze, is_frozen, freezable from immutable import TracingRegion as Region -from immutable import Cown, InterpreterLocal +from immutable import Cown, InterpreterLocal, RegionRef +from test.support import import_helper, os_helper def sort_region_error(msg): """Normalize a 'region could not be closed' message by masking the object @@ -123,7 +124,7 @@ def test_failed_cyclic_region_close(self): with self.assertRaises(RuntimeError) as cm: c.release() - + self.assertEqual( sort_region_error(str(cm.exception)), [ @@ -446,3 +447,541 @@ def __del__(self, loca_bridge=local_bridge, local_medic=local_medic): # Check that the revived medic object is valid self.assertIn("Medic object at 0x", str(local_medic.get())) + +class TestRegionRef(unittest.TestCase): + """A region reference does not keep a region open. It checks on every + dereference whether this interpreter may reach the target, and opens the + region tree on the way.""" + + def _obj(self, tag=0): + @freezable + class A: + pass + obj = A() + obj.tag = tag + return obj + + def test_deref_while_open(self): + r = Region() + r.obj = self._obj(1) + rr = RegionRef(r.obj) + self.assertIs(rr(), r.obj) + + def test_survives_close_unlike_weakref(self): + """A close clears the plain weak references pointing into the region + but re-homes the region references instead.""" + r = Region() + r.obj = self._obj(2) + wref = weakref.ref(r.obj) + rr = RegionRef(r.obj) + + c = Cown(r) + del r + c.release() + + self.assertIsNone(wref()) + c.acquire() + self.assertEqual(rr().tag, 2) + + def test_denied_while_released(self): + r = Region() + r.obj = self._obj(3) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + + with self.assertRaises(RuntimeError) as cm: + rr() + self.assertIn("released cown", str(cm.exception)) + + def test_deref_opens_the_region(self): + r = Region() + r.obj = self._obj(4) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + c.acquire() + + self.assertTrue(c._is_closed()) + obj = rr() + self.assertFalse(c._is_closed()) + self.assertIs(obj, c.value.obj) + + def test_deref_opens_the_whole_chain(self): + """A reference into a nested region has to open every region above it, + not just the one holding the target.""" + child = Region() + child.obj = self._obj(5) + rr = RegionRef(child.obj) + r = Region() + r.child = child + del child + + c = Cown(r) + del r + c.release() + c.acquire() + + self.assertTrue(c._is_closed()) + self.assertEqual(rr().tag, 5) + self.assertFalse(c._is_closed()) + self.assertEqual(c.value.child.obj.tag, 5) + + def test_release_after_acquire_without_opening(self): + """A release does not re-trace an already closed region, so nothing + re-stamps its node. The cown node is what keeps the next owner able to + dereference.""" + r = Region() + r.obj = self._obj(6) + rr = RegionRef(r.obj) + c = Cown(r) + del r + + c.release() + c.acquire() + c.release() + + with self.assertRaises(RuntimeError): + rr() + + c.acquire() + self.assertEqual(rr().tag, 6) + + def test_region_outliving_its_cown(self): + r = Region() + r.obj = self._obj(7) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + c.acquire() + + escaped = c.value + del c + gc.collect() + + self.assertEqual(rr().tag, 7) + self.assertIsNotNone(escaped) + + def test_target_moving_between_regions(self): + """A close re-homes every reference to the objects it traced, so an + object that changed regions resolves through the one it lives in.""" + c1 = Cown(Region()) + c1.value.obj = self._obj(8) + rr = RegionRef(c1.value.obj) + c1.release() + c1.acquire() + + c2 = Cown(Region()) + c2.value.obj = c1.value.obj + c1.value.obj = None + + # Closing the old region should not restrict the region reference + c1.release() + self.assertEqual(rr().tag, 8) + + # Closing the owning reference should restrict the region reference + c2.release() + with self.assertRaises(RuntimeError): + rr() + + # Opening the owning cown allows the region reference again + c2.acquire() + self.assertEqual(rr().tag, 8) + + def test_dead_target(self): + r = Region() + r.obj = self._obj(9) + rr = RegionRef(r.obj) + r.obj = None + gc.collect() + + self.assertIsNone(rr()) + self.assertIn("dead", repr(rr)) + + def test_frozen_target_drops_the_check(self): + """A frozen object is reachable from everywhere, so its references stop + carrying an ownership check.""" + obj = self._obj(10) + rr = RegionRef(obj) + freeze(obj) + self.assertEqual(rr().tag, 10) + + def test_repr_does_not_open_the_region(self): + r = Region() + r.obj = self._obj(11) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + + self.assertIn("unavailable", repr(rr)) + + c.acquire() + self.assertTrue(c._is_closed()) + self.assertIn("to '", repr(rr)) + self.assertTrue(c._is_closed()) + + def test_no_callback_argument(self): + # FIXME(regions): Callbacks are not supported yet. + obj = self._obj(12) + with self.assertRaises(TypeError): + RegionRef(obj, lambda ref: None) + + def test_equality(self): + obj = self._obj(13) + other = self._obj(13) + self.assertEqual(RegionRef(obj), RegionRef(obj)) + self.assertNotEqual(RegionRef(obj), RegionRef(other)) + + def test_failed_close_leaves_the_reference_local(self): + """A close that fails leaves its node unresolved. It has to end up + local to this interpreter, or the reference would be stuck.""" + child = Region() + child.obj = self._obj(15) + rr = RegionRef(child.obj) + r = Region() + r.child = child + del child + + leak = self._obj(16) + r.leak = leak + c = Cown(r) + del r + + with self.assertRaises(RuntimeError): + c.release() + self.assertEqual(rr().tag, 15) + + def test_closed_but_not_released_leaves_the_reference_local(self): + """The tree can close and the cown still refuse to release. Nothing + roots the region in that case, so it stays ours.""" + r = Region() + r.obj = self._obj(17) + rr = RegionRef(r.obj) + c = Cown(r) + del r + + held = c.value + with self.assertRaises(RuntimeError) as cm: + c.release() + self.assertIn("incoming references", str(cm.exception)) + self.assertEqual(rr().tag, 17) + + del held + gc.collect() + c.release() + with self.assertRaises(RuntimeError): + rr() + + def test_deeply_nested_region(self): + inner = Region() + inner.obj = self._obj(18) + rr = RegionRef(inner.obj) + node = inner + for _ in range(30): + outer = Region() + outer.child = node + node = outer + c = Cown(node) + del node, inner, outer + + c.release() + c.acquire() + self.assertEqual(rr().tag, 18) + + def test_repeated_close_open_cycles(self): + """Each close allocates a fresh node and re-homes the reference onto + it. The old ones have to go away with it.""" + r = Region() + r.obj = self._obj(19) + rr = RegionRef(r.obj) + c = Cown(r) + del r + + for _ in range(100): + c.release() + c.acquire() + self.assertEqual(rr().tag, 19) + + def test_target_collected_as_cyclic_garbage(self): + """The collector clears weak references itself, without going through + the type. A RegionRef is not a weakref subtype, so every one of those + paths has to know about it.""" + obj = self._obj(20) + obj.self = obj + rr = RegionRef(obj) + del obj + gc.collect() + self.assertIsNone(rr()) + + def test_reference_itself_is_cyclic_garbage(self): + """A RegionRef that is itself unreachable has to be cleared before the + garbage is deleted, or a __del__ could still dereference it.""" + obj = self._obj(21) + cell = [RegionRef(obj)] + cell.append(cell) + del cell + gc.collect() + self.assertIsNotNone(obj) + + def test_not_freezable(self): + """Freezing would have to either drag the target into the frozen set or + let an immutable object reach a mutable one. Neither is acceptable, so + a reference is simply not freezable -- while its type still is, since + closing a region freezes the type of everything it moves.""" + obj = self._obj(22) + rr = RegionRef(obj) + with self.assertRaises(TypeError): + freeze(rr) + # The type itself stays freezable: closing a region freezes the type of + # everything it moves, so a reference could not live in one otherwise. + freeze(RegionRef) + self.assertTrue(is_frozen(RegionRef)) + # The target must still be able to die cleanly afterwards. + del obj + gc.collect() + self.assertIsNone(rr()) + + def test_already_frozen_target_is_unrestricted(self): + """Freezing before or after the reference is created has to give the + same answer, otherwise the semantics depend on ordering.""" + early = self._obj(23) + freeze(early) + rr_early = RegionRef(early) + + late = self._obj(24) + rr_late = RegionRef(late) + freeze(late) + + self.assertEqual(rr_early().tag, 23) + self.assertEqual(rr_late().tag, 24) + + def test_hash_is_checked_even_when_cached(self): + """A cached hash would otherwise stand as an answer about an object + this interpreter may no longer touch.""" + r = Region() + r.obj = self._obj(26) + rr = RegionRef(r.obj) + hash(rr) + + c = Cown(r) + del r + c.release() + with self.assertRaises(RuntimeError): + hash(rr) + + c.acquire() + self.assertIsInstance(hash(rr), int) + + def test_repr_preserves_a_pending_exception(self): + """repr() runs from error reporting paths, so a denied check must not + wipe the error state the caller is carrying.""" + r = Region() + r.obj = self._obj(25) + rr = RegionRef(r.obj) + c = Cown(r) + del r + c.release() + + try: + raise ValueError("caller's error") + except ValueError: + self.assertIn("unavailable", repr(rr)) + self.assertIsInstance(sys.exception(), ValueError) + + def test_reference_inside_a_region(self): + """A region reference is not followed by the close trace, so storing + one in a region does not drag its target in.""" + outside = self._obj(14) + r = Region() + r.rr = RegionRef(outside) + c = Cown(r) + del r + + c.release() + self.assertFalse(is_frozen(outside)) + c.acquire() + self.assertIs(c.value.rr(), outside) + + +class TestRegionRefSubinterpreters(unittest.TestCase): + """The point of the ownership check: another interpreter may only + dereference what it actually owns.""" + + def setUp(self): + self._interpreters = import_helper.import_module('_interpreters') + + def _run_in_subinterp(self, code, shared=None): + interp = self._interpreters.create() + try: + self._interpreters.run_string(interp, code, shared=shared or {}) + finally: + self._interpreters.destroy(interp) + + def test_frozen_target_reachable_from_everywhere(self): + """A frozen target is shareable, so its references carry no ownership + check at all -- whichever order the freeze and the reference happened + in.""" + @freezable + class A: + pass + + early = A(); early.tag = "early" + freeze(early) + late = A(); late.tag = "late" + + r = Region() + r.early_ref = RegionRef(early) + r.late_ref = RegionRef(late) + freeze(late) + c = Cown(r) + del r + c.release() + + self._run_in_subinterp(""" +c.acquire() +assert c.value.early_ref().tag == "early", "frozen before the ref was created" +assert c.value.late_ref().tag == "late", "frozen after the ref was created" +c.release() +""", shared={"c": c}) + + def test_foreign_deallocation_does_not_transfer_ownership(self): + """A cown is immutable and may live inside a region, so the last + reference to it can be dropped by an interpreter that never owned it. + Its region must go to the cown's owner, not to whoever runs the + deallocator.""" + @freezable + class A: + pass + + inner_region = Region() + inner_region.obj = A() + inner_region.obj.tag = "owned by the creator" + mine = RegionRef(inner_region.obj) + travelling = RegionRef(inner_region.obj) + + inner = Cown(inner_region) + del inner_region + inner.release() + inner.acquire() + # Keep the region alive past the cown, so the cown can die alone. + escaped = inner.value + + outer_region = Region() + outer_region.inner = inner + outer_region.ref = travelling + del travelling + outer = Cown(outer_region) + del outer_region + outer.release() + del inner + gc.collect() + + self._run_in_subinterp(""" +import gc +c.acquire() +c.value.inner = None # drops the last reference to a cown we never owned +gc.collect() +try: + c.value.ref() + raise AssertionError("reached a region owned by another interpreter") +except RuntimeError: + pass +c.release() +""", shared={"c": outer}) + + # The creator still owns it, and is not locked out of its own region. + self.assertEqual(escaped.obj.tag, "owned by the creator") + self.assertEqual(mine().tag, "owned by the creator") + + def test_foreign_deallocation_defers_the_teardown_to_the_owner(self): + """The last reference to a cown owned by this interpreter may be + dropped by another one. The region inside is reference counted + non-atomically and tracked in this interpreter's GC list, so the + teardown has to be handed back here instead of running there.""" + log = os_helper.TESTFN + self.addCleanup(os_helper.unlink, log) + + @freezable + class Marker: + def __del__(self): + with open(self.log, "a") as f: + f.write("region\n") + + inner_region = Region() + inner_region.marker = Marker() + inner_region.marker.log = log + ref = RegionRef(inner_region.marker) + + inner = Cown(inner_region) + del inner_region + inner.release() + inner.acquire() # owned by this interpreter from here on + + outer_region = Region() + outer_region.inner = inner + outer = Cown(outer_region) + del outer_region + outer.release() + del inner + gc.collect() + + with open(log, "w"): + pass + + self._run_in_subinterp(""" +import gc +c.acquire() +c.value.inner = None # drops the last reference to a cown we never owned +gc.collect() +with open(log, "a") as f: + f.write("subinterpreter ") +c.release() +""", shared={"c": outer, "log": log}) + + # The teardown was scheduled here and runs at the next eval breaker. + recorded = [] + for _ in range(10000): + with open(log) as f: + recorded = f.read().split() + if len(recorded) == 2: + break + self.assertEqual(recorded, ["subinterpreter", "region"]) + self.assertIsNone(ref()) + + def test_owner_may_deref_and_others_may_not(self): + """`local` was never in a region, so nothing ever re-homes the + reference to it and it stays local to this interpreter. `owned` travels + with the region and becomes reachable by whoever holds the cown.""" + @freezable + class A: + pass + + local = A() + local.tag = "local" + + r = Region() + r.local_ref = RegionRef(local) + r.owned = A() + r.owned.tag = "owned" + r.owned_ref = RegionRef(r.owned) + c = Cown(r) + del r + c.release() + + self._run_in_subinterp(""" +c.acquire() +try: + c.value.local_ref() + raise AssertionError("reached a foreign local object") +except RuntimeError: + pass +assert c.value.owned_ref().tag == "owned" +c.release() +""", shared={"c": c}) + + c.acquire() + self.assertEqual(c.value.local_ref().tag, "local") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 424044e2466f149..9a00cee20505ff7 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1842,10 +1842,11 @@ class newstyleclass(object): pass # TODO: add check that forces layout of unicodefields # weakref import weakref + # The trailing pointer is `region_ref`, see `Include/cpython/weakrefobject.h`. if support.Py_GIL_DISABLED: - expected = size('2Pln4P') + expected = size('2Pln5P') else: - expected = size('2Pln3P') + expected = size('2Pln4P') check(weakref.ref(int), expected) # weakproxy # XXX diff --git a/Makefile.pre.in b/Makefile.pre.in index d8ae75ed97237e0..15a7f4958719206 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1335,6 +1335,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_complexobject.h \ $(srcdir)/Include/internal/pycore_condvar.h \ $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_cown.h \ $(srcdir)/Include/internal/pycore_critical_section.h \ $(srcdir)/Include/internal/pycore_crossinterp.h \ $(srcdir)/Include/internal/pycore_crossinterp_data_registry.h \ @@ -1362,6 +1363,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_global_strings.h \ $(srcdir)/Include/internal/pycore_hamt.h \ $(srcdir)/Include/internal/pycore_hashtable.h \ + $(srcdir)/Include/internal/pycore_immutability.h \ $(srcdir)/Include/internal/pycore_import.h \ $(srcdir)/Include/internal/pycore_importdl.h \ $(srcdir)/Include/internal/pycore_index_pool.h \ @@ -1412,6 +1414,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pythread.h \ $(srcdir)/Include/internal/pycore_qsbr.h \ $(srcdir)/Include/internal/pycore_range.h \ + $(srcdir)/Include/internal/pycore_regionref.h \ $(srcdir)/Include/internal/pycore_runtime.h \ $(srcdir)/Include/internal/pycore_runtime_init.h \ $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ diff --git a/Modules/_immutablemodule.c b/Modules/_immutablemodule.c index 2c5a5ac665a5108..01127d1b14a0ed8 100644 --- a/Modules/_immutablemodule.c +++ b/Modules/_immutablemodule.c @@ -666,6 +666,17 @@ immutable_exec(PyObject *module) { return -1; } + if (PyModule_AddType(module, &_PyRegionref_RefType) != 0) { + return -1; + } + // The type object itself has to be freezable: the close trace freezes the + // type of every object it moves into a region. Individual references are + // marked unfreezable in `regionref___new__()` instead. + if (_PyImmutability_SetFreezable( + (PyObject*)&_PyRegionref_RefType, _Py_FREEZABLE_YES) < 0) { + return -1; + } + if (PyModule_AddIntConstant(module, "FREEZABLE_YES", _Py_FREEZABLE_YES) != 0) { return -1; diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 2aee8b07891c919..fe8dabaf8ac4c7a 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -484,13 +484,18 @@ _interp_call_pack(PyThreadState *tstate, struct interp_call *call, "expected a callable, got %R", func); return -1; } - if (_PyFunction_GetXIData(tstate, func, &call->_preallocated.func) < 0) { - PyObject *exc = _PyErr_GetRaisedException(tstate); - if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) { - _PyErr_SetRaisedException(tstate, exc); - return -1; + // If func is immutable (e.g. frozen), share it directly instead of + // marshaling its code. + if (_PyObject_GetXIDataNoFallback(tstate, func, &call->_preallocated.func) < 0) { + _PyErr_Clear(tstate); + if (_PyFunction_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + _PyErr_SetRaisedException(tstate, exc); + return -1; + } + Py_DECREF(exc); } - Py_DECREF(exc); } call->func = &call->_preallocated.func; // Handle the args. diff --git a/Objects/cownobject.c b/Objects/cownobject.c index c4dc73843f01e49..fc3304d78b82ce8 100644 --- a/Objects/cownobject.c +++ b/Objects/cownobject.c @@ -1,8 +1,10 @@ #include "Python.h" #include "pymacro.h" +#include "pycore_ceval.h" // _PyEval_AddPendingCall() #include "pycore_cown.h" #include "pycore_immutability.h" +#include "pycore_interp.h" // _PyInterpreterState_LookUpID() #include "pycore_lock.h" #include "pycore_time.h" // _PyTime_FromSeconds() @@ -62,10 +64,26 @@ struct _PyCownObject { PyMutex lock; }; +_PyCown_ipid_t _PyCown_ReleasedIpid(void) { + return RELEASED_IPID; +} + +_PyCown_thread_id_t _PyCown_UnsetThreadId(void) { + return UNSET_THREAD_ID; +} + static _PyCown_ipid_t cown_get_owner(_PyCownObject *obj) { return _Py_atomic_load_uint64(&obj->owning_ip); } +_PyCown_ipid_t _PyCown_Owner(PyObject *cown) { + return cown_get_owner(_PyCownObject_CAST(cown)); +} + +_PyCown_thread_id_t _PyCown_LockingThread(PyObject *cown) { + return _Py_atomic_load_uint64(&_PyCownObject_CAST(cown)->locking_thread); +} + #define BAIL_UNLESS_OWNED_BY(o, owned_by, result) \ do {\ _PyCown_ipid_t owning_ip = cown_get_owner(_PyCownObject_CAST(o)); \ @@ -81,9 +99,26 @@ static _PyCown_ipid_t cown_get_owner(_PyCownObject *obj) { #define BAIL_UNLESS_OWNED_NULL(o) BAIL_UNLESS_OWNED(o, NULL) static int cown_set_value_unchecked(_PyCownObject* self, PyObject* value) { + // Storing a value requires ownership. The exception is the teardown of a + // released cown, which nobody owns and only its last reference can reach. + assert(cown_get_owner(self) == RELEASED_IPID + || cown_get_owner(self) == _PyCown_ThisInterpreterId()); + + // The region is moving out of the cown, so its region references answer to + // the cown's owner from now on. + if (self->value != value && Region_Check(self->value)) { + _PyTracingRegion_SetMetaOwner(self->value, cown_get_owner(self)); + } + // Update the value Py_XSETREF(self->value, Py_NewRef(value)); + // The region is now owned by this cown, so its region references resolve + // through it and follow whoever holds it. + if (Region_Check(value)) { + _PyTracingRegion_SetMetaCown(value, _PyObject_CAST(self)); + } + return 0; } @@ -171,12 +206,11 @@ static int cown_lock(_PyCownObject* self, PyTime_t timeout, _PyCown_ipid_t locki return COWN_ACQUIRE_ERROR; } - // Set the locking thread. - if (has_gil) { - self->locking_thread = _PyCown_ThisThreadId(); - } else { - self->locking_thread = UNSET_THREAD_ID; - } + // Set the locking thread. Stored atomically because `_PyCown_LockingThread()` + // reads it from interpreters that do not own the cown. + _Py_atomic_store_uint64( + &self->locking_thread, + has_gil ? _PyCown_ThisThreadId() : UNSET_THREAD_ID); if (self->value && Region_Check(self->value)) { assert(!PyObject_GC_IsTracked(self->value)); @@ -213,6 +247,7 @@ static int PyCown_init(_PyCownObject *self, PyObject *args, PyObject *kwds) { if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &value)) { return -1; } + self->value = Py_None; // Init the cown as being acquired by the current interpreter _PyCown_ipid_t this_ip = _PyCown_ThisInterpreterId(); @@ -258,17 +293,87 @@ static int PyCown_reachable(_PyCownObject *self, visitproc visit, void *arg) { } static int PyCown_clear(_PyCownObject *self) { + if (_Py_IsImmutable(self->value)) { + Py_CLEAR(self->value); + return 0; + } + + // A mutable value is a region, which may only be dropped by the interpreter + // owning this cown. `PyCown_dealloc` makes sure that this runs there. cown_set_value_unchecked(self, Py_None); - Py_CLEAR(self->value); return 0; } -static void PyCown_dealloc(_PyCownObject *self) { - PyObject_GC_UnTrack(self); +/* Tears the cown down. Only the interpreter owning the cown may run this, see + * `cown_handoff_dealloc`. */ +static void cown_dealloc_owned(_PyCownObject *self) { + // Clearing hands the region off, so no region reference points here any more. PyCown_clear(self); PyObject_GC_Del(self); } +static int cown_pending_dealloc(void *arg) { + cown_dealloc_owned((_PyCownObject *)arg); + return 0; +} + +/* Hands the teardown to the interpreter owning the cown and returns true, or + * returns false when the caller should tear the cown down itself. + * + * A cown is immutable, so the last reference to it can be dropped by an + * interpreter that never owned it. Its region can not be dropped there: while + * the cown is acquired, the region is reference counted non-atomically and + * tracked in the owner's GC list, so touching it would race with the owner. + * + * The cown itself is handed over as well, instead of only its region, because a + * region reference resolving through this cown borrows the pointer. Freeing the + * cown here would leave that pointer dangling until the scheduled call runs. + * + * A cown waiting for its owner is unreachable: its reference count is zero, it + * supports no weak references, and `_Py_TryIncref_Immutable` refuses to + * resurrect it. It must never be revived either, since a revived cown could be + * released a second time, with its region already gone. + */ +static bool cown_handoff_dealloc(_PyCownObject *self) { + _PyCown_ipid_t owner = cown_get_owner(self); + // Nobody owns a released cown, which makes the caller the only one that can + // reach the region. + if (owner == RELEASED_IPID || owner == _PyCown_ThisInterpreterId()) { + return false; + } + + // The lookup raises when the interpreter is gone, and a deallocation can + // happen mid-raise. + PyObject *exc = PyErr_GetRaisedException(); + // FIXME(regions): Can the interpreter go away in the middle of scheduling? + // `weakref_schedule_callbacks` in `Python/immutability.c` asks the same. + PyInterpreterState *target = _PyInterpreterState_LookUpID((int64_t)owner); + bool scheduled = target != NULL + && _PyEval_AddPendingCall(target, cown_pending_dealloc, self, 0) + == _Py_ADD_PENDING_SUCCESS; + PyErr_SetRaisedException(exc); + if (scheduled) { + return true; + } + + // The owner is gone, or its call queue is full. Tearing the cown down here + // is all that is left to do, so the region ends up owned by nobody. An + // interpreter that is already gone can at least not race with us. + _Py_atomic_store_uint64(&self->owning_ip, RELEASED_IPID); + return false; +} + +static void PyCown_dealloc(_PyCownObject *self) { + // Reaching zero returned the cown to this interpreter's GC list. Nothing may + // traverse it, `PyCown_traverse` asserts as much. + PyObject_GC_UnTrack(self); + + if (cown_handoff_dealloc(self)) { + return; + } + cown_dealloc_owned(self); +} + static int lock_acquire_parse_args(PyObject *args, PyObject *kwds, PyTime_t *timeout) @@ -406,8 +511,6 @@ static int cown_close_region(_PyCownObject *self) { return -1; } - // TODO(regions): Test that we can't create weak refs to the bridge object. Otherwise, we also need to clear them. - // The region is closed and this is the only owner of the bridge. We untrack // from the current GC list. PyObject_GC_UnTrack(self->value); @@ -431,6 +534,11 @@ static int cown_release(_PyCownObject *self, _PyCown_ipid_t unlocking_ip) { return -1; } + // The close leaves the region local to this interpreter. Rooting it here, + // after every check has passed, is what lets the next owner of the cown + // dereference the region references pointing into it. + _PyTracingRegion_SetMetaCown(self->value, _PyObject_CAST(self)); + // Region is closed, safe to release return cown_release_unchecked(self, unlocking_ip); } diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c index 49572adcf06f0d5..ee8c7c90028c390 100644 --- a/Objects/tracingregionobject.c +++ b/Objects/tracingregionobject.c @@ -7,6 +7,7 @@ #include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_weakref.h" #include "pycore_cown.h" +#include "pycore_regionref.h" #define ERROR_OBJECT_REPORT_COUNT 5 #define ERROR_MERMAID_REPORT_LIMIT 50 @@ -368,6 +369,13 @@ gc_list_dissolve(PyGC_Head *list) { gc_list_merge(list, &(gc_state->old[0].head)); } +typedef struct { + // The weak references that live inside the region and therefore survive the + // close. NULL when the trace did not find any. + _Py_hashtable_t *keep; + PyObject *region; +} detach_weak_refs_state_t; + static int detach_weak_refs_visit(_Py_hashtable_t *ht, const void *key, const void *value, void *user_data) { @@ -375,6 +383,7 @@ detach_weak_refs_visit(_Py_hashtable_t *ht, const void *key, const void *value, if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(item))) { return 0; } + detach_weak_refs_state_t *state = (detach_weak_refs_state_t *)user_data; #ifdef Py_DEBUG Py_ssize_t weak_ctn = _PyWeakref_GetWeakrefCount(item); @@ -382,24 +391,31 @@ detach_weak_refs_visit(_Py_hashtable_t *ht, const void *key, const void *value, dbg("- Clearing %zd weak references to %p", weak_ctn, item); } #endif - _PyWeakref_ClearWeakRefsExcept(item, (_Py_hashtable_t*)user_data); + _PyRegionRef_CloseWeakRefs(item, state->keep, state->region); return 0; } -/* Detaches all weak references pointing to objects inside the region. +/* Detaches all weak references pointing to objects inside the region, and + * re-homes the region references, which are meant to survive the close. * * This walks the set of traced objects instead of the region's GC list, since * objects that are not tracked by the GC never enter that list. Missing one * would leave a live weak reference pointing into the closed region, which is * enough for external code to read and mutate its contents. + * + * Re-homing rides along on this walk on purpose. Finding the references into a + * region means looking at every member's weakref list, which is exactly what + * this already does. */ -static void detach_weak_refs(_Py_hashtable_t *visited, bool has_weak_refs) { - void* user_data = NULL; - if (has_weak_refs) { - user_data = (void*)visited; - } +static void detach_weak_refs( + PyObject *region, _Py_hashtable_t *visited, bool has_weak_refs) +{ + detach_weak_refs_state_t state = { + .keep = has_weak_refs ? visited : NULL, + .region = region, + }; // `detach_weak_refs_visit()` never fails, so the result can be ignored. - (void)_Py_hashtable_foreach(visited, detach_weak_refs_visit, user_data); + (void)_Py_hashtable_foreach(visited, detach_weak_refs_visit, &state); } typedef struct { @@ -417,12 +433,41 @@ typedef struct { // the region is open. This is the number of references subtracted from the rc. // These are readded in the constructor or when opening the region. Py_ssize_t internal_bridge_refs; + // The node every region reference into this region resolves through, or + // NULL when nothing points into it. Only closed regions can have a meta, + // opening restamps it + _PyRegionRefMetadata *meta; // FIXME(regions): This can be inferred from the status of the gc_list // or stored in the lower bits of the GC list. For now we keep it separate // for the prototype bool open; } TracingRegionObject; +/* Returns this region's node, allocating it if this is the first reference the + * current close has found. Borrowed. The caller must hold `_PyWeakref_Lock`. */ +static _PyRegionRefMetadata * +region_meta_lock_held(TracingRegionObject *self) +{ + if (self->meta == NULL) { + self->meta = _PyRegionRef_NewRegionMetaLockHeld(_PyObject_CAST(self)); + } + return self->meta; +} + +/* Hands the node over to the references still holding it: it becomes local to + * this interpreter with nothing left to open. Used whenever a region stops + * being closed, including when its contents are being deleted. */ +static void +region_meta_release(TracingRegionObject *self) +{ + if (self->meta == NULL) { + return; + } + _PyRegionRef_MetaRegionOpened(self->meta); + _PyRegionRef_MetaDecref(self->meta); + self->meta = NULL; +} + static void _region_close( TracingRegionObject *self, Py_ssize_t bridge_rc, @@ -435,7 +480,7 @@ static void _region_close( dbg("Closing region %p", self); - detach_weak_refs(visited, has_weak_refs); + detach_weak_refs(_PyObject_CAST(self), visited, has_weak_refs); // See comment on `self->internal_bridge_refs` if (bridge_rc != 0) { @@ -470,6 +515,7 @@ static void _open_region(TracingRegionObject *self) { dbg("Opening region %p", self); + region_meta_release(self); _restore_internal_bridge_refs(self); // This only dissolves this region, all sub-regions remain closed. @@ -521,7 +567,7 @@ static void tree_trace_state_destroy(tree_trace_state_t* state) { if (state->hierarchy) { _Py_hashtable_destroy(state->hierarchy); state->hierarchy = NULL; - } + } if (state->pending) { Py_CLEAR(state->pending); } @@ -1338,7 +1384,29 @@ static int _trace_visit_bridge_ref(PyObject* obj, region_trace_state_t* state) { // If the child region is closed we can move it directly if (_PyTracingRegion_IsClosed(obj)) { - return _move_obj(obj, state); + int res = _move_obj(obj, state); + // Update the region reference meta of the child, if it has one + if (res == 0) { + TracingRegionObject *child = (TracingRegionObject *)obj; + bool no_memory = false; + LOCK_REGION_REF_META(); + if (child->meta != NULL) { + _PyRegionRefMetadata *parent = + region_meta_lock_held((TracingRegionObject *)state->bridge); + if (parent == NULL) { + no_memory = true; + } + else { + _PyRegionRef_MetaSetParentLockHeld(child->meta, parent); + } + } + UNLOCK_REGION_REF_META(); + if (no_memory) { + PyErr_NoMemory(); + return TRACE_RES_ERR; + } + } + return res; } _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(tree_state->hierarchy, (void*)obj); @@ -1517,6 +1585,27 @@ static int _try_close_region(PyObject *region_obj, tree_trace_state_t *tree_trac return region_trace_res; } +/* Resolves the region reference meta of every region this trace touched. + */ +static int +resolve_region_meta(_Py_hashtable_t *ht, const void *key, const void *value, + void *user_data) +{ + TracingRegionObject *region = (TracingRegionObject *)key; + if (region->meta == NULL) { + return 0; + } + // The region remains open, therefore we mark it as being local to the IP + if (region->open) { + region_meta_release(region); + return 0; + } + + // The region was closed, we resolve the WIP state + _PyRegionRef_MetaResolveWip(region->meta); + return 0; +} + static int try_close_region_tree(PyObject *root) { dbg("Starting region tree trace from %p", root); @@ -1586,6 +1675,8 @@ static int try_close_region_tree(PyObject *root) { error: tree_trace_res = TRACE_RES_ERR; finally: + // `resolve_region_meta()` never fails, so the result can be ignored. + (void)_Py_hashtable_foreach(state.tracing_counts, resolve_region_meta, NULL); report_missing_reachable(&state); tree_trace_state_destroy(&state); @@ -1606,6 +1697,7 @@ TracingRegion_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { // The region is set up here rather than in `tp_init()`, so that a region // can never be observed in an uninitialized state. gc_list_init(&self->gc_list); + self->meta = NULL; // We make the region open by default, this ensures that the first close // will handle the region type correctly. Alternatively, we could make them // closed in the beginning, but then handle the cases specifically. @@ -1650,18 +1742,15 @@ static void _region_delete_contents(TracingRegionObject *self) { gc_list_init(&members); gc_list_init(&survivors); - // Steal the members and open the region first. A finalizer reaching the - // bridge calls `_open_region()`, which would otherwise dissolve the very - // list being disposed of here. + // Steal the members first. `_open_region()` will then set internal values + // but keep not invalidate `members`. gc_list_merge(&self->gc_list, &members); assert(gc_list_is_empty(&self->gc_list)); - // Has to happen before anything is released, the members still hold these. - _restore_internal_bridge_refs(self); - self->open = true; + _open_region(self); // The disposal needs a clean error state; a dealloc can happen mid-raise. PyObject *exc = PyErr_GetRaisedException(); - + // Finalize everything before anything is released, so that no `__del__` // observes a member that is already gone. _PyGC_FinalizeGarbage(&members); @@ -1826,6 +1915,35 @@ int _PyTracingRegion_IsClosed(PyObject* region) { return !self->open; } +/* Opens the region, so that a region reference can hand out a strong reference + * into it. Only the sub-regions of this region stay closed. + * + * This function requires the GIL to be held, and the caller to have established + * that this interpreter owns the region. + */ +void _PyTracingRegion_Open(PyObject* region) { + _open_region((TracingRegionObject*)region); +} + +_PyRegionRefMetadata *_PyTracingRegion_MetaLockHeld(PyObject* region) { + return region_meta_lock_held((TracingRegionObject*)region); +} + +void _PyTracingRegion_SetMetaCown(PyObject* region, PyObject* cown) { + TracingRegionObject *self = (TracingRegionObject*)region; + // Meta is only set if the region is closed and has region references + if (self->meta != NULL) { + _PyRegionRef_MetaSetCown(self->meta, cown); + } +} + +void _PyTracingRegion_SetMetaOwner(PyObject* region, _PyCown_ipid_t owner) { + TracingRegionObject *self = (TracingRegionObject*)region; + if (self->meta != NULL) { + _PyRegionRef_MetaSetIpid(self->meta, owner); + } +} + static PyMethodDef TracingRegion_methods[] = { {NULL, NULL} /* sentinel */ }; @@ -1853,5 +1971,3 @@ PyTypeObject _PyTracingRegion_Type = { .tp_finalize = TracingRegion_finalize, .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, }; - -// TODO: RegionReferences diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index ba1acff39d4b6ea..0ba2119ed92992a 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -6,6 +6,10 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "pycore_cown.h" // _PyCown_ThisInterpreterId() +#include "pycore_immutability.h" // _PyTracingRegion_Open() +#include "pycore_interp.h" // PyInterpreterState.immutability +#include "pycore_regionref.h" // FIXME(region): Reusing the same weakref easily breaks region isolation // without a simple way out for programmers. For now we disable the optimization. @@ -113,7 +117,476 @@ _PyWeakref_GetWeakrefCount(PyObject *obj) return count; } +// ################################################################### +// Region reference metadata +// ################################################################### + +/* See `pycore_regionref.h` for the design. + * + * Every field of a `_PyRegionRefMetadata` is guarded by `_PyWeakref_Lock`. The + * `_lock_held` helpers below expect the caller to hold it. + */ + +/* In the default build the weakref list lock and the metadata lock are the same + * global mutex, so code holding the former must not take the latter again. In + * free-threaded builds they differ. */ +#ifdef Py_GIL_DISABLED +# define LOCK_META_UNDER_WEAKREFS() LOCK_REGION_REF_META() +# define UNLOCK_META_UNDER_WEAKREFS() UNLOCK_REGION_REF_META() +#else +# define LOCK_META_UNDER_WEAKREFS() ((void)0) +# define UNLOCK_META_UNDER_WEAKREFS() ((void)0) +#endif + +static void clear_weakref_lock_held(PyWeakReference *self, PyObject **callback); + +static _PyRegionRefMetadata * +meta_new_lock_held(uint8_t kind) +{ + // Raw allocation on purpose: a node can outlive the interpreter that + // created it, when a chain reaching it is still held elsewhere. + _PyRegionRefMetadata *meta = PyMem_RawMalloc(sizeof(_PyRegionRefMetadata)); + if (meta == NULL) { + return NULL; + } + meta->rc = 1; + meta->kind = kind; + meta->region = NULL; + memset(&meta->value, 0, sizeof(meta->value)); + return meta; +} + +static void +meta_incref_lock_held(_PyRegionRefMetadata *meta) +{ + if (meta != NULL) { + assert(meta->rc > 0); + meta->rc += 1; + } +} + +static void +meta_decref_lock_held(_PyRegionRefMetadata *meta) +{ + // Releasing a node releases its parent, and a chain is as deep as the + // region tree. Walk it instead of recursing. + while (meta != NULL) { + assert(meta->rc > 0); + if (--meta->rc > 0) { + return; + } + _PyRegionRefMetadata *parent = NULL; + if (meta->kind == _Py_REGION_REF_META) { + parent = meta->value.parent; + } + PyMem_RawFree(meta); + meta = parent; + } +} + +/* Returns a terminal node owned by this interpreter, for a reference to an + * object that is in no region. New reference, NULL when out of memory. */ +static _PyRegionRefMetadata * +meta_new_local_lock_held(void) +{ + _PyRegionRefMetadata *meta = meta_new_lock_held(_Py_REGION_REF_IPID); + if (meta != NULL) { + meta->value.ipid = _PyCown_ThisInterpreterId(); + } + return meta; +} + +/* Releases whatever the node delegated to. The callers below all assign the new + * kind and value right after, so the stale union is never observed. */ +static void +meta_clear_parent_lock_held(_PyRegionRefMetadata *meta) +{ + if (meta->kind == _Py_REGION_REF_META) { + _PyRegionRefMetadata *parent = meta->value.parent; + meta->kind = _Py_REGION_REF_WIP; + meta_decref_lock_held(parent); + } +} + +static void +meta_set_parent_lock_held(_PyRegionRefMetadata *meta, _PyRegionRefMetadata *parent) +{ + assert(meta != NULL && parent != NULL); + assert(meta != parent); + + // Increfing first keeps a self-assignment from freeing the parent. + meta_incref_lock_held(parent); + meta_clear_parent_lock_held(meta); + meta->kind = _Py_REGION_REF_META; + meta->value.parent = parent; +} + +static void +meta_set_cown_lock_held(_PyRegionRefMetadata *meta, PyObject *cown) +{ + meta_clear_parent_lock_held(meta); + meta->kind = _Py_REGION_REF_COWN; + meta->value.cown = cown; +} + +static void +meta_set_ipid_lock_held(_PyRegionRefMetadata *meta, _PyCown_ipid_t ipid) +{ + meta_clear_parent_lock_held(meta); + meta->kind = _Py_REGION_REF_IPID; + meta->value.ipid = ipid; +} + +static void +set_region_ref_lock_held(PyWeakReference *self, _PyRegionRefMetadata *meta) +{ + // FIXME(regions): Why does this fail? assert(_PyRegionRef_CheckExact(self)); + if (self->region_ref == meta) { + return; + } + meta_incref_lock_held(meta); + meta_decref_lock_held(self->region_ref); + self->region_ref = meta; +} + +/* Drops a reference's metadata while its weakref list lock is held. */ +static void +clear_region_ref_lock_held(PyWeakReference *self) +{ + // FIXME(regions): Why does this fail? assert(_PyRegionRef_CheckExact(self)); + LOCK_META_UNDER_WEAKREFS(); + set_region_ref_lock_held(self, NULL); + UNLOCK_META_UNDER_WEAKREFS(); +} + +_PyRegionRefMetadata * +_PyRegionRef_NewRegionMetaLockHeld(PyObject *region) +{ + _PyRegionRefMetadata *meta = meta_new_lock_held(_Py_REGION_REF_WIP); + if (meta != NULL) { + meta->region = region; + } + return meta; +} + +void +_PyRegionRef_MetaDecref(_PyRegionRefMetadata *meta) +{ + LOCK_REGION_REF_META(); + meta_decref_lock_held(meta); + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_MetaSetParentLockHeld(_PyRegionRefMetadata *meta, + _PyRegionRefMetadata *parent) +{ + meta_set_parent_lock_held(meta, parent); +} + +void +_PyRegionRef_MetaSetCown(_PyRegionRefMetadata *meta, PyObject *cown) +{ + LOCK_REGION_REF_META(); + meta_set_cown_lock_held(meta, cown); + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_MetaSetIpid(_PyRegionRefMetadata *meta, _PyCown_ipid_t ipid) +{ + // FIXME(regions): `ipid` should always be the current interpreter. It isn't + // for a released cown, or when `PyCown_clear` runs on an interpreter that + // doesn't own the cown; once that is refactored this can assert it. + LOCK_REGION_REF_META(); + meta_set_ipid_lock_held(meta, ipid); + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_MetaRegionOpened(_PyRegionRefMetadata *meta) +{ + LOCK_REGION_REF_META(); + meta->region = NULL; + meta_set_ipid_lock_held(meta, _PyCown_ThisInterpreterId()); + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_MetaResolveWip(_PyRegionRefMetadata *meta) +{ + LOCK_REGION_REF_META(); + if (meta->kind == _Py_REGION_REF_WIP) { + meta_set_ipid_lock_held(meta, _PyCown_ThisInterpreterId()); + } + UNLOCK_REGION_REF_META(); +} + +void +_PyRegionRef_CloseWeakRefs(PyObject *obj, _Py_hashtable_t *keep, PyObject *region) +{ + PyWeakReference **list = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(obj); + LOCK_WEAKREFS(obj); + LOCK_META_UNDER_WEAKREFS(); + while (*list) { + PyWeakReference *ref = *list; + + // Region references remain in the list, but their region reference meta + // is repointed. + if (_PyRegionRef_CheckExact((PyObject *)ref)) { + _PyRegionRefMetadata *meta = _PyTracingRegion_MetaLockHeld(region); + if (meta == NULL) { + // Out of memory, we clear the reference and continue + clear_weakref_lock_held(ref, NULL); + continue; + } + set_region_ref_lock_held(ref, meta); + list = &ref->wr_next; + } + else if (keep != NULL && _Py_hashtable_get_entry(keep, ref)) { + list = &ref->wr_next; + } + else { + clear_weakref_lock_held(ref, NULL); + } + } + UNLOCK_META_UNDER_WEAKREFS(); + UNLOCK_WEAKREFS(obj); +} + +// ################################################################### +// Region reference access +// ################################################################### + +/* The regions a dereference has to open, innermost first. Most chains are + * shallow, so the common case stays on the stack. */ +#define REGIONREF_OPEN_STACK 8 + +typedef struct { + PyObject **items; + Py_ssize_t count; + Py_ssize_t capacity; + PyObject *stack[REGIONREF_OPEN_STACK]; +} regionref_open_list_t; + +static void +open_list_init(regionref_open_list_t *list) +{ + list->items = list->stack; + list->count = 0; + list->capacity = REGIONREF_OPEN_STACK; +} + +static void +open_list_clear(regionref_open_list_t *list) +{ + if (list->items != list->stack) { + PyMem_RawFree(list->items); + } + open_list_init(list); +} + +/* Grows with the raw allocator so this stays safe to call under the metadata + * lock, which must not run Python code. */ +static int +open_list_push(regionref_open_list_t *list, PyObject *region) +{ + if (list->count == list->capacity) { + Py_ssize_t capacity = list->capacity * 2; + PyObject **items; + if (list->items == list->stack) { + items = PyMem_RawMalloc(capacity * sizeof(PyObject *)); + if (items != NULL) { + memcpy(items, list->stack, list->count * sizeof(PyObject *)); + } + } + else { + items = PyMem_RawRealloc(list->items, capacity * sizeof(PyObject *)); + } + if (items == NULL) { + return -1; + } + list->items = items; + list->capacity = capacity; + } + list->items[list->count++] = region; + return 0; +} + +typedef enum { + REGIONREF_ALLOWED, + REGIONREF_DENIED_WIP, + REGIONREF_DENIED_IPID, + REGIONREF_DENIED_COWN, + REGIONREF_DENIED_MEMORY, +} regionref_verdict_t; + +/* Resolves the reference's metadata chain and decides whether this interpreter + * may reach the target. On success `regions` lists the regions that still have + * to be opened, innermost first; pass NULL to only ask the question. + * + * Returns 0 when access is allowed, -1 with an exception set otherwise. + */ +static int +regionref_check_access(PyWeakReference *self, regionref_open_list_t *regions, + bool quiet) +{ + const _PyCown_ipid_t this_ip = _PyCown_ThisInterpreterId(); + regionref_verdict_t verdict = REGIONREF_ALLOWED; + _PyCown_ipid_t owner = 0; + _PyCown_thread_id_t locking_thread = 0; + bool wrong_thread = false; + + // Nothing inside this section may raise or allocate through Python. + LOCK_REGION_REF_META(); + _PyRegionRefMetadata *meta = self->region_ref; + while (meta != NULL) { + if (meta->region != NULL && regions != NULL) { + if (open_list_push(regions, meta->region) < 0) { + verdict = REGIONREF_DENIED_MEMORY; + break; + } + } + if (meta->kind != _Py_REGION_REF_META) { + break; + } + meta = meta->value.parent; + } + if (verdict == REGIONREF_ALLOWED && meta != NULL) { + switch (meta->kind) { + case _Py_REGION_REF_WIP: + verdict = REGIONREF_DENIED_WIP; + break; + case _Py_REGION_REF_IPID: + owner = meta->value.ipid; + if (owner != this_ip) { + verdict = REGIONREF_DENIED_IPID; + } + break; + case _Py_REGION_REF_COWN: + owner = _PyCown_Owner(meta->value.cown); + if (owner != this_ip) { + verdict = REGIONREF_DENIED_COWN; + } + else { + locking_thread = _PyCown_LockingThread(meta->value.cown); + wrong_thread = locking_thread != _PyCown_UnsetThreadId() + && locking_thread != _PyCown_ThisThreadId(); + } + break; + default: + Py_UNREACHABLE(); + } + } + // A NULL node means the target was frozen, which makes it reachable from + // everywhere. Every live region reference has a node from birth. + UNLOCK_REGION_REF_META(); + + if (wrong_thread) { + // FIXME(regions): Thread ownership is not enforced, any thread of the + // owning interpreter may reach the data. Whether that should change is + // a question for once this has seen some use. + fprintf(stderr, + "RegionRef dereferenced from thread %llu, but the cown was " + "acquired by thread %llu\n", + (unsigned long long)_PyCown_ThisThreadId(), + (unsigned long long)locking_thread); + } + + if (verdict == REGIONREF_ALLOWED) { + return 0; + } + if (quiet) { + // Callers that only want the answer. Raising here and having them + // clear it would destroy whatever the caller already had pending; + // `repr()` in particular runs from error reporting paths. + return -1; + } + + switch (verdict) { + case REGIONREF_ALLOWED: + return 0; + case REGIONREF_DENIED_MEMORY: + PyErr_NoMemory(); + return -1; + case REGIONREF_DENIED_WIP: + PyErr_SetString( + PyExc_RuntimeError, + "the region holding this reference is currently being closed"); + return -1; + case REGIONREF_DENIED_COWN: + if (owner == _PyCown_ReleasedIpid()) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %llu attempted to dereference a region reference " + "into a released cown", + (unsigned long long)this_ip); + return -1; + } + _Py_FALLTHROUGH; + case REGIONREF_DENIED_IPID: + if (owner == _PyCown_ReleasedIpid()) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %llu attempted to dereference a region reference " + "into a region that no interpreter owns", + (unsigned long long)this_ip); + return -1; + } + PyErr_Format( + PyExc_RuntimeError, + "interpreter %llu attempted to dereference a region reference " + "into a region owned by %llu", + (unsigned long long)this_ip, (unsigned long long)owner); + return -1; + } + Py_UNREACHABLE(); +} + +/* Returns a new strong reference to the target. + * + * Returns NULL without an exception when the target simply died, and NULL with + * one set when this interpreter may not reach it. + */ +static PyObject * +regionref_get_ref(PyObject *op) +{ + PyWeakReference *self = _PyWeakref_CAST(op); + + // A dead target needs no ownership check; it is not in any region any more. + if (_Py_atomic_load_ptr(&self->wr_object) == Py_None) { + return NULL; + } + + regionref_open_list_t regions; + open_list_init(®ions); + if (regionref_check_access(self, ®ions, false) < 0) { + open_list_clear(®ions); + return NULL; + } + + // Opening runs no Python code but does move GC lists, so it happens with + // the metadata lock dropped. Parent regions first, so an open region never has a + // closed ancestor. The borrowed region pointers stay valid because the + // check above established that this interpreter owns them, and only an + // owner can deallocate a region. + for (Py_ssize_t i = regions.count - 1; i >= 0; i--) { + _PyTracingRegion_Open(regions.items[i]); + } + open_list_clear(®ions); + + PyObject *obj = _Py_atomic_load_ptr(&self->wr_object); + if (obj == Py_None) { + return NULL; + } + LOCK_WEAKREFS(obj); + PyObject *result = get_ref_lock_held(self, obj); + UNLOCK_WEAKREFS(obj); + return result; +} + static PyObject *weakref_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject *regionref_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames); static void init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) @@ -129,12 +602,16 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) else { self->callback_ipid = PyInterpreterState_GetID(PyInterpreterState_Get()); } - self->vectorcall = weakref_vectorcall; + // A region reference has to run its ownership check before handing out the + // target, so it cannot share the plain weakref fast path. + self->vectorcall = _PyRegionRef_CheckExact((PyObject *)self) + ? regionref_vectorcall : weakref_vectorcall; #ifdef Py_GIL_DISABLED self->weakrefs_lock = &WEAKREF_LIST_LOCK(ob); _PyObject_SetMaybeWeakref(ob); _PyObject_SetMaybeWeakref((PyObject *)self); #endif + self->region_ref = NULL; } // Clear the weakref and steal its callback into `callback`, if provided. @@ -163,6 +640,7 @@ clear_weakref_lock_held(PyWeakReference *self, PyObject **callback) *callback = self->wr_callback; self->wr_callback = NULL; } + clear_region_ref_lock_held(self); } // Clear the weakref and its callback @@ -196,8 +674,13 @@ void _PyWeakref_ClearRef(PyWeakReference *self) { assert(self != NULL); - assert(PyWeakref_Check(self)); + // Region references reuse this struct without being a weakref subtype. + assert(_PyWeakrefOrRegionRef_Check(self)); + // Callers here hold no lock, but `region_ref` needs one. Callers that + // already hold it use `clear_weakref_lock_held()` directly. + LOCK_REGION_REF_META(); clear_weakref_lock_held(self, NULL); + UNLOCK_REGION_REF_META(); } static void @@ -224,8 +707,12 @@ gc_clear(PyObject *op) PyWeakReference *self = _PyWeakref_CAST(op); PyObject *callback; // The world is stopped during GC in free-threaded builds. It's safe to - // call this without holding the lock. + // call this without holding the list lock. `region_ref` still needs the + // metadata lock in the default build, where each interpreter has its own + // GIL and the collector is not alone. + LOCK_REGION_REF_META(); clear_weakref_lock_held(self, &callback); + UNLOCK_REGION_REF_META(); Py_XDECREF(callback); return 0; } @@ -489,11 +976,16 @@ _PyWeakref_OnObjectFreeze(PyObject *object) return; } LOCK_WEAKREFS(object); + LOCK_META_UNDER_WEAKREFS(); PyWeakReference *current = *list; while (current != NULL) { + // A frozen object is reachable from every interpreter, so a region + // reference to it no longer needs an ownership check. + set_region_ref_lock_held(current, NULL); immutable_make_weakref_safe(current); current = current->wr_next; } + UNLOCK_META_UNDER_WEAKREFS(); UNLOCK_WEAKREFS(object); } @@ -626,6 +1118,235 @@ _PyWeakref_RefType = { .tp_free = PyObject_GC_Del, }; +// ################################################################### +// Region reference type +// ################################################################### + +/* A region reference is a weak reference that survives its target's region + * being closed. Instead of keeping the region open, every dereference asks + * whether the target may be reached and opens the region tree if it + * may. + * + * This type is deliberately not a subtype of `_PyWeakref_RefType`, which is + * what keeps `PyWeakref_Check()` false for it. + */ + +static PyObject * +regionref_vectorcall(PyObject *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + if (!_PyArg_NoKwnames("RegionRef", kwnames)) { + return NULL; + } + if (!_PyArg_CheckPositional("RegionRef", PyVectorcall_NARGS(nargsf), 0, 0)) { + return NULL; + } + PyObject *obj = regionref_get_ref(self); + if (obj == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; + } + return obj; +} + +static Py_hash_t +regionref_hash(PyObject *op) +{ + PyWeakReference *self = _PyWeakref_CAST(op); + // Checked before the cache is consulted: a cached hash would otherwise be + // a standing answer about an object this interpreter may no longer touch. + if (regionref_check_access(self, NULL, false) < 0) { + return -1; + } + Py_hash_t hash = _Py_atomic_load_ssize_relaxed(&self->hash); + if (hash != -1) { + return hash; + } + PyObject *obj = regionref_get_ref(op); + if (obj == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, "weak object has gone away"); + } + return -1; + } + hash = PyObject_Hash(obj); + Py_DECREF(obj); + _Py_atomic_store_ssize_relaxed(&self->hash, hash); + return hash; +} + +static PyObject * +regionref_repr(PyObject *self) +{ + PyWeakReference *ref = _PyWeakref_CAST(self); + PyObject *obj = _Py_atomic_load_ptr(&ref->wr_object); + if (obj == Py_None) { + return PyUnicode_FromFormat(""); + } + + // Deliberately only asks the question instead of going through + // `regionref_get_ref()`, so that printing a reference never opens a + // region, matching `TracingRegion`'s repr. + if (regionref_check_access(ref, NULL, true) < 0) { + return PyUnicode_FromFormat(""); + } + + // This needs a lock, since the object may be in the middle of finalizing when this + // is being called. + LOCK_WEAKREFS(obj); + PyObject *target = get_ref_lock_held(ref, obj); + UNLOCK_WEAKREFS(obj); + if (target == NULL) { + return PyUnicode_FromFormat(""); + } + PyObject *repr = PyUnicode_FromFormat( + "", self, target, target); + Py_DECREF(target); + return repr; +} + +/* Region references only support equality, and compare by target like weak + * references do. A reference whose target is gone or out of reach falls back to + * identity, since there is nothing to compare. */ +static PyObject * +regionref_richcompare(PyObject *self, PyObject *other, int op) +{ + if ((op != Py_EQ && op != Py_NE) + || !_PyRegionRef_CheckExact(self) + || !_PyRegionRef_CheckExact(other)) + { + Py_RETURN_NOTIMPLEMENTED; + } + + // An unreachable target compares by identity, like a dead one. The check + // runs quietly so that a denial never disturbs the caller's error state. + PyObject *obj = NULL; + PyObject *other_obj = NULL; + if (regionref_check_access(_PyWeakref_CAST(self), NULL, true) == 0) { + obj = regionref_get_ref(self); + } + if (regionref_check_access(_PyWeakref_CAST(other), NULL, true) == 0) { + other_obj = regionref_get_ref(other); + } + if (PyErr_Occurred()) { + Py_XDECREF(obj); + Py_XDECREF(other_obj); + return NULL; + } + + if (obj == NULL || other_obj == NULL) { + Py_XDECREF(obj); + Py_XDECREF(other_obj); + int res = (self == other); + if (op == Py_NE) { + res = !res; + } + return PyBool_FromLong(res); + } + + PyObject *res = PyObject_RichCompare(obj, other_obj, op); + Py_DECREF(obj); + Py_DECREF(other_obj); + return res; +} + +static PyObject * +regionref___new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + // FIXME(regions): Region references do not support callbacks yet. Adding + // them means deciding which interpreter runs the callback and how. + if (!_PyArg_NoKeywords("RegionRef", kwargs)) { + return NULL; + } + PyObject *ob; + if (!PyArg_UnpackTuple(args, "__new__", 1, 1, &ob)) { + return NULL; + } + + PyWeakReference *ref = get_or_create_weakref(type, ob, NULL); + if (ref == NULL) { + return NULL; + } + + // FIXME(region): Freezing a region reference needs special handling like weak + // references. We also need to handle a case, where a freeze would propagate into + // a closed region. The solution is probably a pre-freeze hook that calls freeze + // on the target object. + if (_PyImmutability_SetFreezable( + (PyObject *)ref, _Py_FREEZABLE_NO) < 0) { + Py_DECREF(ref); + return NULL; + } + + // An immutable target is reachable from everywhere, no meta is set. + if (_Py_IsImmutable(ob)) { + return (PyObject *)ref; + } + + // Until a close re-homes it, the target is local to this interpreter. + LOCK_REGION_REF_META(); + _PyRegionRefMetadata *meta = meta_new_local_lock_held(); + if (meta != NULL) { + set_region_ref_lock_held(ref, meta); + meta_decref_lock_held(meta); + } + UNLOCK_REGION_REF_META(); + if (meta == NULL) { + Py_DECREF(ref); + return PyErr_NoMemory(); + } + + return (PyObject *)ref; +} + +static int +regionref___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (!_PyArg_NoKeywords("RegionRef", kwargs)) { + return -1; + } + PyObject *tmp; + return PyArg_UnpackTuple(args, "__init__", 1, 1, &tmp) ? 0 : -1; +} + +PyDoc_STRVAR(regionref_doc, +"RegionRef(object)\n\ +--\n\ +\n\ +A weak reference into a region that does not keep the region open.\n\ +Calling it returns the referenced object if th object may be reached,\n\ +or raises a RuntimeError otherwise. Returns None once the referenced \n\ +object is gone."); + +PyTypeObject +_PyRegionref_RefType = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "immutable.RegionRef", + .tp_basicsize = sizeof(PyWeakReference), + .tp_dealloc = weakref_dealloc, + .tp_vectorcall_offset = offsetof(PyWeakReference, vectorcall), + .tp_call = PyVectorcall_Call, + .tp_repr = regionref_repr, + .tp_hash = regionref_hash, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_HAVE_VECTORCALL, + .tp_doc = regionref_doc, + .tp_traverse = gc_traverse, + // tp_reachable explicitly doesn't visit the weak reference to reflect the + // actual RC of referenced objects. Changes to this will require adjustments + // in freezing and region traversal code. + .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, + .tp_clear = gc_clear, + .tp_richcompare = regionref_richcompare, + .tp_methods = weakref_methods, + .tp_init = regionref___init__, + .tp_alloc = PyType_GenericAlloc, + .tp_new = regionref___new__, + .tp_free = PyObject_GC_Del, +}; + static bool proxy_check_ref(PyObject *obj) @@ -1277,7 +1998,7 @@ _PyWeakref_ClearWeakRefsExcept(PyObject *obj, _Py_hashtable_t *keep) if (keep != NULL && _Py_hashtable_get_entry(keep, *list)) { list = &((*list)->wr_next); } else { - _PyWeakref_ClearRef(*list); + clear_weakref_lock_held(*list, NULL); } } UNLOCK_WEAKREFS(obj); diff --git a/Python/gc.c b/Python/gc.c index 67d2a6fcb01262c..4bd73e4143db9a5 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1061,10 +1061,11 @@ clear_weakrefs(PyGC_Head *unreachable) PyObject *op = FROM_GC(gc); next = GC_NEXT(gc); - if (PyWeakref_Check(op)) { + if (_PyWeakrefOrRegionRef_Check(op)) { /* A weakref inside the unreachable set is always cleared. See * the comments above handle_weakref_callbacks() for why these - * must be cleared. + * must be cleared. Region references reuse the same struct + * without being a weakref subtype, and need it just as much. */ _PyWeakref_ClearRef((PyWeakReference *)op); } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index b08d74b1c2f8a3a..1f97f94c297e830 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1590,8 +1590,10 @@ clear_weakrefs(struct collection_state *state) { PyObject *op; WORKSTACK_FOR_EACH(&state->unreachable, op) { - if (PyWeakref_Check(op)) { - // Clear weakrefs that are themselves unreachable. + if (_PyWeakrefOrRegionRef_Check(op)) { + // Clear weakrefs that are themselves unreachable. Region + // references reuse the same struct without being a weakref + // subtype, and need clearing just as much. _PyWeakref_ClearRef((PyWeakReference *)op); }