diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index 92d32937872d..707524c61097 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -652,11 +652,15 @@ def generate_len_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: ) emitter.emit_error_check("retval", fn.ret_type, "return -1;") if is_int_rprimitive(fn.ret_type): - emitter.emit_line("Py_ssize_t val = CPyTagged_AsSsize_t(retval);") + emitter.emit_line("Py_ssize_t val = CPyTagged_AsLen(retval);") + emitter.emit_dec_ref("retval", fn.ret_type) else: - emitter.emit_line("Py_ssize_t val = PyLong_AsSsize_t(retval);") - emitter.emit_dec_ref("retval", fn.ret_type) - emitter.emit_line("if (PyErr_Occurred()) return -1;") + # CPy_LenFromResult() takes an object, so box other unboxed types such as i64 + emitter.emit_box("retval", "retbox", fn.ret_type, declare_dest=True) + if fn.ret_type.is_unboxed: + emitter.emit_line("if (unlikely(retbox == NULL)) return -1;") + emitter.emit_line("Py_ssize_t val = CPy_LenFromResult(retbox);") + emitter.emit_dec_ref("retbox", object_rprimitive) emitter.emit_line("return val;") emitter.emit_line("}") diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 5fa291a6ae8e..9ce358408f91 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -152,8 +152,10 @@ from mypyc.primitives.generic_ops import ( call_fastcall_wrapper_op, call_legacy_wrapper_op, + check_len_result_op, generic_len_op, generic_ssize_t_len_op, + len_from_result_op, py_call_op, py_call_with_kwargs_op, py_call_with_posargs_op, @@ -2963,18 +2965,16 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val # TODO: Support use_pyssize_t assert not use_pyssize_t length = self.gen_method_call(val, "__len__", [], int_rprimitive, line) + rtype = length.type + if not ( + is_tagged(rtype) or is_fixed_width_rtype(rtype) or is_bool_or_bit_rprimitive(rtype) + ): + # Like len(), accept any int (or object with __index__) that is a valid length + obj = self.coerce(length, object_rprimitive, line) + return self.call_c(len_from_result_op, [obj], line) length = self.coerce(length, int_rprimitive, line) - ok, fail = BasicBlock(), BasicBlock() - cond = self.binary_op(length, Integer(0), ">=", line) - self.add_bool_branch(cond, ok, fail) - self.activate_block(fail) - self.add( - RaiseStandardError( - RaiseStandardError.VALUE_ERROR, "__len__() should return >= 0", line - ) - ) - self.add(Unreachable()) - self.activate_block(ok) + # The length must not be negative and must fit in Py_ssize_t + self.call_c(check_len_result_op, [length], line) return length elif isinstance(typ, RVec): diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 5151bb7cd745..4904b1a69b8f 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -925,6 +925,27 @@ static void CPy_LogGetAttr(const char *method, PyObject *obj, PyObject *attr) { CPyTagged CPyObject_Hash(PyObject *o); Py_hash_t CPy_HashFromResult(PyObject *res); +Py_ssize_t CPy_LenFromResult(PyObject *res); +CPyTagged CPyObject_LenFromResult(PyObject *res); + +// Convert the result of a __len__ method to a length, like len() does. +// Return -1 with an exception set on error. +static inline Py_ssize_t CPyTagged_AsLen(CPyTagged x) { + if (likely(CPyTagged_CheckShort(x))) { + Py_ssize_t len = CPyTagged_ShortAsSsize_t(x); + if (likely(len >= 0)) { + return len; + } + PyErr_SetString(PyExc_ValueError, "__len__() should return >= 0"); + return -1; + } + return CPy_LenFromResult(CPyTagged_LongAsObject(x)); +} + +// Check that the result of a __len__ method is a valid length, like len() does +static inline bool CPyTagged_CheckLen(CPyTagged x) { + return CPyTagged_AsLen(x) >= 0; +} // Default identity-based hash (same as object.__hash__), used as tp_hash static inline Py_hash_t CPy_DefaultHash(PyObject *o) { diff --git a/mypyc/lib-rt/generic_ops.c b/mypyc/lib-rt/generic_ops.c index d421b53ebf0a..e267f104c982 100644 --- a/mypyc/lib-rt/generic_ops.c +++ b/mypyc/lib-rt/generic_ops.c @@ -37,6 +37,33 @@ Py_hash_t CPy_HashFromResult(PyObject *res) { return h == -1 ? -2 : h; } +// Convert the result of a __len__ method to a length like slot_sq_length in CPython: +// the result must be an int (or have __index__), not negative, and fit in Py_ssize_t. +// Return -1 with an exception set on error. +Py_ssize_t CPy_LenFromResult(PyObject *res) { + PyObject *index = PyNumber_Index(res); + if (index == NULL) { + return -1; + } + if (PyLong_IsNegative(index)) { + Py_DECREF(index); + PyErr_SetString(PyExc_ValueError, "__len__() should return >= 0"); + return -1; + } + Py_ssize_t len = PyNumber_AsSsize_t(index, PyExc_OverflowError); + Py_DECREF(index); + return len; +} + +// Like CPy_LenFromResult, but return the length as a tagged int +CPyTagged CPyObject_LenFromResult(PyObject *res) { + Py_ssize_t len = CPy_LenFromResult(res); + if (len < 0) { + return CPY_INT_TAG; + } + return CPyTagged_FromSsize_t(len); +} + PyObject *CPyObject_GetAttr3(PyObject *v, PyObject *name, PyObject *defl) { PyObject *result; diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 1355619211f9..a6a8a1c4ae2c 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -11,8 +11,9 @@ from __future__ import annotations -from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER +from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( + bit_rprimitive, bool_rprimitive, c_int_rprimitive, c_pointer_rprimitive, @@ -374,6 +375,22 @@ error_kind=ERR_NEG_INT, ) +# Check that the result of a native __len__ method is a valid length, like len() does +check_len_result_op = custom_op( + arg_types=[int_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyTagged_CheckLen", + error_kind=ERR_FALSE, +) + +# Convert the result of a native __len__ method that isn't an int to a length +len_from_result_op = custom_op( + arg_types=[object_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyObject_LenFromResult", + error_kind=ERR_MAGIC, +) + # iter(obj) iter_op = function_op( name="builtins.iter", diff --git a/mypyc/test-data/irbuild-dunders.test b/mypyc/test-data/irbuild-dunders.test index 1796a7e2160e..9e2d34437cf7 100644 --- a/mypyc/test-data/irbuild-dunders.test +++ b/mypyc/test-data/irbuild-dunders.test @@ -16,17 +16,36 @@ def f(c): c :: __main__.C r0 :: int r1 :: bit - r2 :: bool L0: r0 = c.__len__() - r1 = int_ge r0, 0 - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise ValueError('__len__() should return >= 0') - unreachable -L2: + r1 = CPyTagged_CheckLen(r0) return r0 +[case testDundersLenAny] +from typing import Any + +class C: + def __len__(self) -> Any: + return 2 + +def f(c: C) -> int: + return len(c) +[out] +def C.__len__(self): + self :: __main__.C + r0 :: object +L0: + r0 = object 2 + return r0 +def f(c): + c :: __main__.C + r0 :: object + r1 :: int +L0: + r0 = c.__len__() + r1 = CPyObject_LenFromResult(r0) + return r1 + [case testDundersSetItem] class C: def __setitem__(self, key: int, value: int) -> None: diff --git a/mypyc/test-data/run-dunders.test b/mypyc/test-data/run-dunders.test index 4f932b684c8d..127cc82a6f93 100644 --- a/mypyc/test-data/run-dunders.test +++ b/mypyc/test-data/run-dunders.test @@ -1523,6 +1523,124 @@ def test_hash_result() -> None: with assertRaises(TypeError, "__hash__ method should return an integer"): hash(HashAny(x)) +[case testDundersLenResult] +# len() checks the value returned by __len__ like in Python +from typing import Any + +from mypy_extensions import i64, i8, u8 +from testutil import assertRaises + +class Len: + def __init__(self, n: int) -> None: + self.n = n + + def __len__(self) -> int: + return self.n + +class LenAny: + def __init__(self, n: Any) -> None: + self.n = n + + def __len__(self) -> Any: + return self.n + +class LenBool: + def __init__(self, n: bool) -> None: + self.n = n + + def __len__(self) -> bool: + return self.n + +class LenI64: + def __init__(self, n: i64) -> None: + self.n = n + + def __len__(self) -> i64: + return self.n + +class LenI8: + def __init__(self, n: i8) -> None: + self.n = n + + def __len__(self) -> i8: + return self.n + +class LenU8: + def __init__(self, n: u8) -> None: + self.n = n + + def __len__(self) -> u8: + return self.n + +class Index: + def __index__(self) -> int: + return 7 + +def len_any(x: Any) -> int: + # Uses the sq_length/mp_length slot + return len(x) + +NEGATIVE = "__len__() should return >= 0" +TOO_LARGE = "cannot fit 'int' into an index-sized integer" + +def test_len_int() -> None: + for n in [0, 5, 2**30, 2**31 - 1]: + assert len(Len(n)) == n + assert len_any(Len(n)) == n + for n in [-1, -5, -(2**63), -(2**100)]: + with assertRaises(ValueError, NEGATIVE): + len(Len(n)) + with assertRaises(ValueError, NEGATIVE): + len_any(Len(n)) + for n in [2**63, 2**100]: + with assertRaises(OverflowError, TOO_LARGE): + len(Len(n)) + with assertRaises(OverflowError, TOO_LARGE): + len_any(Len(n)) + +def test_len_any() -> None: + for x, n in [(3, 3), (True, 1), (Index(), 7)]: + assert len(LenAny(x)) == n + assert len_any(LenAny(x)) == n + for x in [-1, -(2**100)]: + with assertRaises(ValueError, NEGATIVE): + len(LenAny(x)) + with assertRaises(ValueError, NEGATIVE): + len_any(LenAny(x)) + with assertRaises(OverflowError, TOO_LARGE): + len(LenAny(2**63)) + with assertRaises(OverflowError, TOO_LARGE): + len_any(LenAny(2**63)) + for x in ["x", 1.5, None]: + with assertRaises(TypeError, "cannot be interpreted as an integer"): + len(LenAny(x)) + with assertRaises(TypeError, "cannot be interpreted as an integer"): + len_any(LenAny(x)) + +def test_len_bool() -> None: + assert len(LenBool(True)) == 1 + assert len(LenBool(False)) == 0 + assert len_any(LenBool(True)) == 1 + assert len_any(LenBool(False)) == 0 + +def test_len_native_int() -> None: + for n in [0, 7, 2**31 - 1]: + assert len(LenI64(n)) == n + assert len_any(LenI64(n)) == n + for n in [-1, -(2**63)]: + with assertRaises(ValueError, NEGATIVE): + len(LenI64(n)) + with assertRaises(ValueError, NEGATIVE): + len_any(LenI64(n)) + assert len(LenI8(127)) == 127 + assert len_any(LenI8(127)) == 127 + with assertRaises(ValueError, NEGATIVE): + len(LenI8(-128)) + with assertRaises(ValueError, NEGATIVE): + len_any(LenI8(-128)) + assert len(LenU8(255)) == 255 + assert len_any(LenU8(255)) == 255 + [case testDundersImplicitNeWithNarrowRhs] class Accepted: pass