From 6583e4335e02bc6f0368cc1f7d3cda869fd04af0 Mon Sep 17 00:00:00 2001 From: Jahongir Qurbonov Date: Sun, 15 Nov 2026 03:34:04 +0500 Subject: [PATCH 1/2] [mypyc] Check the result of native __len__ methods like Python The sq_length/mp_length slot of a native class returned a negative __len__ result without setting an exception, so len() raised SystemError or crashed. With a bool or native int return type, the slot passed the C value to PyLong_AsSsize_t(), which doesn't compile. With an Any return type, objects with __index__ were rejected. The slot now converts the result like slot_sq_length in CPython, and the compiled len() of a native class instance also raises OverflowError for a length that doesn't fit in Py_ssize_t. --- mypyc/codegen/emitwrapper.py | 18 +++- mypyc/irbuild/ll_builder.py | 22 ++--- mypyc/lib-rt/CPy.h | 14 ++++ mypyc/lib-rt/generic_ops.c | 27 ++++++ mypyc/primitives/generic_ops.py | 19 ++++- mypyc/test-data/irbuild-dunders.test | 33 ++++++-- mypyc/test-data/run-dunders.test | 118 +++++++++++++++++++++++++++ 7 files changed, 228 insertions(+), 23 deletions(-) diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index 92d32937872d9..91e61502f04d6 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -652,11 +652,21 @@ def generate_len_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: ) emitter.emit_error_check("retval", fn.ret_type, "return -1;") if is_int_rprimitive(fn.ret_type): - emitter.emit_line("Py_ssize_t val = CPyTagged_AsSsize_t(retval);") + emitter.emit_line("if (likely(CPyTagged_CheckShort(retval))) {") + emitter.emit_line("Py_ssize_t val = CPyTagged_ShortAsSsize_t(retval);") + emitter.emit_line("if (likely(val >= 0)) return val;") + emitter.emit_line('PyErr_SetString(PyExc_ValueError, "__len__() should return >= 0");') + emitter.emit_line("return -1;") + emitter.emit_line("}") + emitter.emit_line("Py_ssize_t val = CPy_LenFromResult(CPyTagged_LongAsObject(retval));") + emitter.emit_dec_ref("retval", fn.ret_type) else: - emitter.emit_line("Py_ssize_t val = PyLong_AsSsize_t(retval);") - emitter.emit_dec_ref("retval", fn.ret_type) - emitter.emit_line("if (PyErr_Occurred()) return -1;") + # CPy_LenFromResult() takes an object, so box other unboxed types such as i64 + emitter.emit_box("retval", "retbox", fn.ret_type, declare_dest=True) + if fn.ret_type.is_unboxed: + emitter.emit_line("if (unlikely(retbox == NULL)) return -1;") + emitter.emit_line("Py_ssize_t val = CPy_LenFromResult(retbox);") + emitter.emit_dec_ref("retbox", object_rprimitive) emitter.emit_line("return val;") emitter.emit_line("}") diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 5fa291a6ae8ec..9ce358408f915 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -152,8 +152,10 @@ from mypyc.primitives.generic_ops import ( call_fastcall_wrapper_op, call_legacy_wrapper_op, + check_len_result_op, generic_len_op, generic_ssize_t_len_op, + len_from_result_op, py_call_op, py_call_with_kwargs_op, py_call_with_posargs_op, @@ -2963,18 +2965,16 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val # TODO: Support use_pyssize_t assert not use_pyssize_t length = self.gen_method_call(val, "__len__", [], int_rprimitive, line) + rtype = length.type + if not ( + is_tagged(rtype) or is_fixed_width_rtype(rtype) or is_bool_or_bit_rprimitive(rtype) + ): + # Like len(), accept any int (or object with __index__) that is a valid length + obj = self.coerce(length, object_rprimitive, line) + return self.call_c(len_from_result_op, [obj], line) length = self.coerce(length, int_rprimitive, line) - ok, fail = BasicBlock(), BasicBlock() - cond = self.binary_op(length, Integer(0), ">=", line) - self.add_bool_branch(cond, ok, fail) - self.activate_block(fail) - self.add( - RaiseStandardError( - RaiseStandardError.VALUE_ERROR, "__len__() should return >= 0", line - ) - ) - self.add(Unreachable()) - self.activate_block(ok) + # The length must not be negative and must fit in Py_ssize_t + self.call_c(check_len_result_op, [length], line) return length elif isinstance(typ, RVec): diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 5151bb7cd745e..684b316dc8071 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -925,6 +925,20 @@ static void CPy_LogGetAttr(const char *method, PyObject *obj, PyObject *attr) { CPyTagged CPyObject_Hash(PyObject *o); Py_hash_t CPy_HashFromResult(PyObject *res); +Py_ssize_t CPy_LenFromResult(PyObject *res); +CPyTagged CPyObject_LenFromResult(PyObject *res); + +// Check that the result of a __len__ method is a valid length, like len() does +static inline bool CPyTagged_CheckLen(CPyTagged x) { + if (likely(CPyTagged_CheckShort(x))) { + if (likely(CPyTagged_ShortAsSsize_t(x) >= 0)) { + return true; + } + PyErr_SetString(PyExc_ValueError, "__len__() should return >= 0"); + return false; + } + return CPy_LenFromResult(CPyTagged_LongAsObject(x)) >= 0; +} // Default identity-based hash (same as object.__hash__), used as tp_hash static inline Py_hash_t CPy_DefaultHash(PyObject *o) { diff --git a/mypyc/lib-rt/generic_ops.c b/mypyc/lib-rt/generic_ops.c index d421b53ebf0ad..e267f104c9828 100644 --- a/mypyc/lib-rt/generic_ops.c +++ b/mypyc/lib-rt/generic_ops.c @@ -37,6 +37,33 @@ Py_hash_t CPy_HashFromResult(PyObject *res) { return h == -1 ? -2 : h; } +// Convert the result of a __len__ method to a length like slot_sq_length in CPython: +// the result must be an int (or have __index__), not negative, and fit in Py_ssize_t. +// Return -1 with an exception set on error. +Py_ssize_t CPy_LenFromResult(PyObject *res) { + PyObject *index = PyNumber_Index(res); + if (index == NULL) { + return -1; + } + if (PyLong_IsNegative(index)) { + Py_DECREF(index); + PyErr_SetString(PyExc_ValueError, "__len__() should return >= 0"); + return -1; + } + Py_ssize_t len = PyNumber_AsSsize_t(index, PyExc_OverflowError); + Py_DECREF(index); + return len; +} + +// Like CPy_LenFromResult, but return the length as a tagged int +CPyTagged CPyObject_LenFromResult(PyObject *res) { + Py_ssize_t len = CPy_LenFromResult(res); + if (len < 0) { + return CPY_INT_TAG; + } + return CPyTagged_FromSsize_t(len); +} + PyObject *CPyObject_GetAttr3(PyObject *v, PyObject *name, PyObject *defl) { PyObject *result; diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 1355619211f96..a6a8a1c4ae2c8 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -11,8 +11,9 @@ from __future__ import annotations -from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER +from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( + bit_rprimitive, bool_rprimitive, c_int_rprimitive, c_pointer_rprimitive, @@ -374,6 +375,22 @@ error_kind=ERR_NEG_INT, ) +# Check that the result of a native __len__ method is a valid length, like len() does +check_len_result_op = custom_op( + arg_types=[int_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyTagged_CheckLen", + error_kind=ERR_FALSE, +) + +# Convert the result of a native __len__ method that isn't an int to a length +len_from_result_op = custom_op( + arg_types=[object_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyObject_LenFromResult", + error_kind=ERR_MAGIC, +) + # iter(obj) iter_op = function_op( name="builtins.iter", diff --git a/mypyc/test-data/irbuild-dunders.test b/mypyc/test-data/irbuild-dunders.test index 1796a7e2160e4..9e2d34437cf76 100644 --- a/mypyc/test-data/irbuild-dunders.test +++ b/mypyc/test-data/irbuild-dunders.test @@ -16,17 +16,36 @@ def f(c): c :: __main__.C r0 :: int r1 :: bit - r2 :: bool L0: r0 = c.__len__() - r1 = int_ge r0, 0 - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise ValueError('__len__() should return >= 0') - unreachable -L2: + r1 = CPyTagged_CheckLen(r0) return r0 +[case testDundersLenAny] +from typing import Any + +class C: + def __len__(self) -> Any: + return 2 + +def f(c: C) -> int: + return len(c) +[out] +def C.__len__(self): + self :: __main__.C + r0 :: object +L0: + r0 = object 2 + return r0 +def f(c): + c :: __main__.C + r0 :: object + r1 :: int +L0: + r0 = c.__len__() + r1 = CPyObject_LenFromResult(r0) + return r1 + [case testDundersSetItem] class C: def __setitem__(self, key: int, value: int) -> None: diff --git a/mypyc/test-data/run-dunders.test b/mypyc/test-data/run-dunders.test index 4f932b684c8da..127cc82a6f933 100644 --- a/mypyc/test-data/run-dunders.test +++ b/mypyc/test-data/run-dunders.test @@ -1523,6 +1523,124 @@ def test_hash_result() -> None: with assertRaises(TypeError, "__hash__ method should return an integer"): hash(HashAny(x)) +[case testDundersLenResult] +# len() checks the value returned by __len__ like in Python +from typing import Any + +from mypy_extensions import i64, i8, u8 +from testutil import assertRaises + +class Len: + def __init__(self, n: int) -> None: + self.n = n + + def __len__(self) -> int: + return self.n + +class LenAny: + def __init__(self, n: Any) -> None: + self.n = n + + def __len__(self) -> Any: + return self.n + +class LenBool: + def __init__(self, n: bool) -> None: + self.n = n + + def __len__(self) -> bool: + return self.n + +class LenI64: + def __init__(self, n: i64) -> None: + self.n = n + + def __len__(self) -> i64: + return self.n + +class LenI8: + def __init__(self, n: i8) -> None: + self.n = n + + def __len__(self) -> i8: + return self.n + +class LenU8: + def __init__(self, n: u8) -> None: + self.n = n + + def __len__(self) -> u8: + return self.n + +class Index: + def __index__(self) -> int: + return 7 + +def len_any(x: Any) -> int: + # Uses the sq_length/mp_length slot + return len(x) + +NEGATIVE = "__len__() should return >= 0" +TOO_LARGE = "cannot fit 'int' into an index-sized integer" + +def test_len_int() -> None: + for n in [0, 5, 2**30, 2**31 - 1]: + assert len(Len(n)) == n + assert len_any(Len(n)) == n + for n in [-1, -5, -(2**63), -(2**100)]: + with assertRaises(ValueError, NEGATIVE): + len(Len(n)) + with assertRaises(ValueError, NEGATIVE): + len_any(Len(n)) + for n in [2**63, 2**100]: + with assertRaises(OverflowError, TOO_LARGE): + len(Len(n)) + with assertRaises(OverflowError, TOO_LARGE): + len_any(Len(n)) + +def test_len_any() -> None: + for x, n in [(3, 3), (True, 1), (Index(), 7)]: + assert len(LenAny(x)) == n + assert len_any(LenAny(x)) == n + for x in [-1, -(2**100)]: + with assertRaises(ValueError, NEGATIVE): + len(LenAny(x)) + with assertRaises(ValueError, NEGATIVE): + len_any(LenAny(x)) + with assertRaises(OverflowError, TOO_LARGE): + len(LenAny(2**63)) + with assertRaises(OverflowError, TOO_LARGE): + len_any(LenAny(2**63)) + for x in ["x", 1.5, None]: + with assertRaises(TypeError, "cannot be interpreted as an integer"): + len(LenAny(x)) + with assertRaises(TypeError, "cannot be interpreted as an integer"): + len_any(LenAny(x)) + +def test_len_bool() -> None: + assert len(LenBool(True)) == 1 + assert len(LenBool(False)) == 0 + assert len_any(LenBool(True)) == 1 + assert len_any(LenBool(False)) == 0 + +def test_len_native_int() -> None: + for n in [0, 7, 2**31 - 1]: + assert len(LenI64(n)) == n + assert len_any(LenI64(n)) == n + for n in [-1, -(2**63)]: + with assertRaises(ValueError, NEGATIVE): + len(LenI64(n)) + with assertRaises(ValueError, NEGATIVE): + len_any(LenI64(n)) + assert len(LenI8(127)) == 127 + assert len_any(LenI8(127)) == 127 + with assertRaises(ValueError, NEGATIVE): + len(LenI8(-128)) + with assertRaises(ValueError, NEGATIVE): + len_any(LenI8(-128)) + assert len(LenU8(255)) == 255 + assert len_any(LenU8(255)) == 255 + [case testDundersImplicitNeWithNarrowRhs] class Accepted: pass From 9c8bbad0623f3420366f0815bd6058991a063740 Mon Sep 17 00:00:00 2001 From: Jahongir Qurbonov Date: Fri, 20 Nov 2026 00:10:23 +0500 Subject: [PATCH 2/2] [mypyc] Share the __len__ result check between len() and the slot wrapper Move the conversion into CPyTagged_AsLen, which returns the length as a Py_ssize_t (-1 on error). The sq_length/mp_length wrapper calls it instead of duplicating the short int fast path, and CPyTagged_CheckLen (used by compiled len()) is now a thin wrapper around it, so the IR is unchanged. --- mypyc/codegen/emitwrapper.py | 8 +------- mypyc/lib-rt/CPy.h | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index 91e61502f04d6..707524c61097a 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -652,13 +652,7 @@ def generate_len_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: ) emitter.emit_error_check("retval", fn.ret_type, "return -1;") if is_int_rprimitive(fn.ret_type): - emitter.emit_line("if (likely(CPyTagged_CheckShort(retval))) {") - emitter.emit_line("Py_ssize_t val = CPyTagged_ShortAsSsize_t(retval);") - emitter.emit_line("if (likely(val >= 0)) return val;") - emitter.emit_line('PyErr_SetString(PyExc_ValueError, "__len__() should return >= 0");') - emitter.emit_line("return -1;") - emitter.emit_line("}") - emitter.emit_line("Py_ssize_t val = CPy_LenFromResult(CPyTagged_LongAsObject(retval));") + emitter.emit_line("Py_ssize_t val = CPyTagged_AsLen(retval);") emitter.emit_dec_ref("retval", fn.ret_type) else: # CPy_LenFromResult() takes an object, so box other unboxed types such as i64 diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 684b316dc8071..4904b1a69b8fd 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -928,16 +928,23 @@ Py_hash_t CPy_HashFromResult(PyObject *res); Py_ssize_t CPy_LenFromResult(PyObject *res); CPyTagged CPyObject_LenFromResult(PyObject *res); -// Check that the result of a __len__ method is a valid length, like len() does -static inline bool CPyTagged_CheckLen(CPyTagged x) { +// Convert the result of a __len__ method to a length, like len() does. +// Return -1 with an exception set on error. +static inline Py_ssize_t CPyTagged_AsLen(CPyTagged x) { if (likely(CPyTagged_CheckShort(x))) { - if (likely(CPyTagged_ShortAsSsize_t(x) >= 0)) { - return true; + Py_ssize_t len = CPyTagged_ShortAsSsize_t(x); + if (likely(len >= 0)) { + return len; } PyErr_SetString(PyExc_ValueError, "__len__() should return >= 0"); - return false; + return -1; } - return CPy_LenFromResult(CPyTagged_LongAsObject(x)) >= 0; + return CPy_LenFromResult(CPyTagged_LongAsObject(x)); +} + +// Check that the result of a __len__ method is a valid length, like len() does +static inline bool CPyTagged_CheckLen(CPyTagged x) { + return CPyTagged_AsLen(x) >= 0; } // Default identity-based hash (same as object.__hash__), used as tp_hash