From 9037c7d61e8bfe38f72b35ec0b169bc27058a9a1 Mon Sep 17 00:00:00 2001 From: Jahongir Qurbonov Date: Tue, 10 Nov 2026 23:38:49 +0500 Subject: [PATCH] [mypyc] Add primitives for int and str containment in list and tuple --- mypyc/doc/tuple_operations.rst | 1 + mypyc/lib-rt/CPy.h | 2 + mypyc/lib-rt/list_ops.c | 106 ++++++++++++++++++ mypyc/primitives/list_ops.py | 24 ++++ mypyc/primitives/tuple_ops.py | 34 +++++- mypyc/test-data/irbuild-lists.test | 70 ++++++++++-- mypyc/test-data/irbuild-tuple.test | 31 ++++++ mypyc/test-data/run-lists.test | 172 +++++++++++++++++++++++++++++ 8 files changed, 430 insertions(+), 10 deletions(-) diff --git a/mypyc/doc/tuple_operations.rst b/mypyc/doc/tuple_operations.rst index 4c9da9b894af6..2b1671113ebfe 100644 --- a/mypyc/doc/tuple_operations.rst +++ b/mypyc/doc/tuple_operations.rst @@ -23,6 +23,7 @@ Operators * ``tup[n:m]``, ``tup[n:]``, ``tup[:m]`` (slicing) * ``tup1 + tup2`` * ``tup * n``, ``n * tup`` +* ``obj in tup`` Statements ---------- diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index d4bd2d8938ce1..2c6ed0a7cadaa 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -997,6 +997,8 @@ PyObject *CPyList_Build(Py_ssize_t len, ...); PyObject *CPyList_PopLast(PyObject *obj); PyObject *CPyList_Pop(PyObject *obj, CPyTagged index); CPyTagged CPyList_Count(PyObject *obj, PyObject *value); +int CPySequence_ContainsInt(PyObject *seq, CPyTagged value); +int CPySequence_ContainsStr(PyObject *seq, PyObject *str); int CPyList_Insert(PyObject *list, CPyTagged index, PyObject *value); PyObject *CPyList_Extend(PyObject *o1, PyObject *o2); int CPyList_Remove(PyObject *list, PyObject *obj); diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index 86caa34a3f5a5..7706d6d0aa9dc 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -396,6 +396,112 @@ CPyTagged CPyList_Count(PyObject *obj, PyObject *value) return list_count((PyListObject *)obj, value); } +// Return 1 if an exact list or tuple contains a short int value, 0 if it doesn't, or +// -1 on error. Exact int items are compared without boxing the value. The boxed value +// is only created if there is an item of some other type. +// +// The caller must hold a critical section on a list. +static int _CPySequence_ContainsShortInt(PyObject *seq, Py_ssize_t value) { + PyObject *boxed = NULL; + int result = 0; + // A list may be resized by __eq__, so the size and item array must be read again + // after each comparison + for (Py_ssize_t i = 0; i < Py_SIZE(seq); i++) { + PyObject *item = PySequence_Fast_GET_ITEM(seq, i); + if (PyLong_CheckExact(item)) { + int overflow; + Py_ssize_t n = CPyLong_AsSsize_tAndOverflow(item, &overflow); + if (n == value && !overflow) { + result = 1; + break; + } + continue; + } + if (boxed == NULL) { + boxed = PyLong_FromSsize_t(value); + if (boxed == NULL) { + result = -1; + break; + } + } + Py_INCREF(item); + int cmp = PyObject_RichCompareBool(item, boxed, Py_EQ); + Py_DECREF(item); + if (cmp != 0) { + result = cmp; + break; + } + } + Py_XDECREF(boxed); + return result; +} + +int CPySequence_ContainsInt(PyObject *seq, CPyTagged value) { + if (likely(CPyTagged_CheckShort(value))) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(value); + if (PyList_CheckExact(seq)) { + int result; + Py_BEGIN_CRITICAL_SECTION(seq); + result = _CPySequence_ContainsShortInt(seq, n); + Py_END_CRITICAL_SECTION(); + return result; + } else if (PyTuple_CheckExact(seq)) { + return _CPySequence_ContainsShortInt(seq, n); + } + } + PyObject *obj = CPyTagged_AsObject(value); + int result = PySequence_Contains(seq, obj); + Py_DECREF(obj); + return result; +} + +// Return 1 if an exact list or tuple contains an exact str, 0 if it doesn't, or -1 on +// error. Exact str items are compared directly. +// +// The caller must hold a critical section on a list. +static int _CPySequence_ContainsExactStr(PyObject *seq, PyObject *str) { + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + int kind = PyUnicode_KIND(str); + const void *data = PyUnicode_DATA(str); + for (Py_ssize_t i = 0; i < Py_SIZE(seq); i++) { + PyObject *item = PySequence_Fast_GET_ITEM(seq, i); + int cmp; + if (item == str) { + return 1; + } else if (PyUnicode_CheckExact(item)) { + // Strings are always in the most compact representation, so equal strings + // have the same kind + cmp = PyUnicode_GET_LENGTH(item) == len + && PyUnicode_KIND(item) == kind + && memcmp(PyUnicode_DATA(item), data, len * kind) == 0; + } else { + Py_INCREF(item); + cmp = PyObject_RichCompareBool(item, str, Py_EQ); + Py_DECREF(item); + } + if (cmp != 0) { + return cmp; + } + } + return 0; +} + +int CPySequence_ContainsStr(PyObject *seq, PyObject *str) { + // A str subclass could override __eq__, so it must use the generic implementation + if (likely(PyUnicode_CheckExact(str))) { + if (PyList_CheckExact(seq)) { + int result; + Py_BEGIN_CRITICAL_SECTION(seq); + result = _CPySequence_ContainsExactStr(seq, str); + Py_END_CRITICAL_SECTION(); + return result; + } else if (PyTuple_CheckExact(seq)) { + return _CPySequence_ContainsExactStr(seq, str); + } + } + return PySequence_Contains(seq, str); +} + int CPyList_Insert(PyObject *list, CPyTagged index, PyObject *value) { if (CPyTagged_CheckShort(index)) { diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index d141a55d057c0..480e14b0bd5ee 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -5,6 +5,7 @@ from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( bit_rprimitive, + bool_rprimitive, c_int_rprimitive, c_pyssize_t_rprimitive, int64_rprimitive, @@ -13,6 +14,7 @@ object_rprimitive, pointer_rprimitive, short_int_rprimitive, + str_rprimitive, uint64_rprimitive, void_rtype, ) @@ -273,6 +275,28 @@ error_kind=ERR_MAGIC, ) +# int in list +binary_op( + name="in", + arg_types=[int_rprimitive, list_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPySequence_ContainsInt", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, + ordering=[1, 0], +) + +# str in list +binary_op( + name="in", + arg_types=[str_rprimitive, list_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPySequence_ContainsStr", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, + ordering=[1, 0], +) + # list.insert(index, obj) method_op( name="insert", diff --git a/mypyc/primitives/tuple_ops.py b/mypyc/primitives/tuple_ops.py index 09daf5396bf0c..7269e96ffc94c 100644 --- a/mypyc/primitives/tuple_ops.py +++ b/mypyc/primitives/tuple_ops.py @@ -9,14 +9,24 @@ from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( bit_rprimitive, + bool_rprimitive, + c_int_rprimitive, c_pyssize_t_rprimitive, int_rprimitive, list_rprimitive, object_rprimitive, + str_rprimitive, tuple_rprimitive, void_rtype, ) -from mypyc.primitives.registry import binary_op, custom_op, function_op, load_address_op, method_op +from mypyc.primitives.registry import ( + ERR_NEG_INT, + binary_op, + custom_op, + function_op, + load_address_op, + method_op, +) # Get the 'builtins.tuple' type object. load_address_op(name="builtins.tuple", type=object_rprimitive, src="PyTuple_Type") @@ -109,6 +119,28 @@ error_kind=ERR_MAGIC, ) +# int in tuple +binary_op( + name="in", + arg_types=[int_rprimitive, tuple_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPySequence_ContainsInt", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, + ordering=[1, 0], +) + +# str in tuple +binary_op( + name="in", + arg_types=[str_rprimitive, tuple_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPySequence_ContainsStr", + error_kind=ERR_NEG_INT, + truncated_type=bool_rprimitive, + ordering=[1, 0], +) + # tuple * int binary_op( name="*", diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 9587ae1c41e6a..8a6da8ab764dc 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -343,22 +343,74 @@ L0: return r0 [case testListIn] -from typing import List +from typing import List, Optional def f(x: List[int], y: int) -> bool: return y in x +def str_in(x: List[str], y: str) -> bool: + return y in x +def not_in(x: List[str], y: str) -> bool: + return y not in x +def generic(x: List[object], y: object, z: Optional[int]) -> bool: + return y in x or z in x [out] def f(x, y): x :: list y :: int - r0 :: object - r1 :: i32 - r2 :: bit - r3 :: bool + r0 :: i32 + r1 :: bit + r2 :: bool L0: - r0 = box(int, y) - r1 = PySequence_Contains(x, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool + r0 = CPySequence_ContainsInt(x, y) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + return r2 +def str_in(x, y): + x :: list + y :: str + r0 :: i32 + r1 :: bit + r2 :: bool +L0: + r0 = CPySequence_ContainsStr(x, y) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + return r2 +def not_in(x, y): + x :: list + y :: str + r0 :: i32 + r1 :: bit + r2, r3 :: bool +L0: + r0 = CPySequence_ContainsStr(x, y) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + r3 = r2 ^ 1 + return r3 +def generic(x, y, z): + x :: list + y :: object + z :: union[int, None] + r0 :: i32 + r1 :: bit + r2, r3 :: bool + r4 :: i32 + r5 :: bit + r6 :: bool +L0: + r0 = PySequence_Contains(x, y) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + if r2 goto L1 else goto L2 :: bool +L1: + r3 = r2 + goto L3 +L2: + r4 = PySequence_Contains(x, z) + r5 = r4 >= 0 :: signed + r6 = truncate r4: i32 to builtins.bool + r3 = r6 +L3: return r3 [case testListInsert] diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 56a57b04a6b07..8b3bccbf0a5a6 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -178,6 +178,37 @@ L2: return r3 +[case testVariableLengthTupleIn] +from typing import Tuple +def int_in(x: Tuple[int, ...], y: int) -> bool: + return y in x +def str_not_in(x: Tuple[str, ...], y: str) -> bool: + return y not in x +[out] +def int_in(x, y): + x :: tuple + y :: int + r0 :: i32 + r1 :: bit + r2 :: bool +L0: + r0 = CPySequence_ContainsInt(x, y) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + return r2 +def str_not_in(x, y): + x :: tuple + y :: str + r0 :: i32 + r1 :: bit + r2, r3 :: bool +L0: + r0 = CPySequence_ContainsStr(x, y) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + r3 = r2 ^ 1 + return r3 + [case testTupleOperatorIn] def f(i: int) -> bool: return i in [1, 2, 3] diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index 03bc24da0b586..8a320a69bc0a8 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -851,3 +851,175 @@ class GoodList(list): return GoodList(list(self) * n + [4]) def copy(self) -> Any: return GoodList(list(self) + [5]) + +[case testListAndTupleContains] +from typing import Any, List, Tuple +from testutil import assertRaises +from contains_helpers import ( + AlwaysEqual, IntSub, IntEqOverride, StrSub, StrEqOverride, ListContainsOverride, TupleContainsOverride, + RaisingEq, Shrinker, NAN, DECIMAL_ONE +) + +def int_in_list(x: int, a: List[Any]) -> bool: + return x in a + +def int_not_in_list(x: int, a: List[Any]) -> bool: + return x not in a + +def str_in_list(x: str, a: List[Any]) -> bool: + return x in a + +def str_not_in_list(x: str, a: List[Any]) -> bool: + return x not in a + +def int_in_tuple(x: int, a: Tuple[Any, ...]) -> bool: + return x in a + +def str_in_tuple(x: str, a: Tuple[Any, ...]) -> bool: + return x in a + +BIG = [2**30 - 1, 2**30, 2**31, 2**62 - 1, 2**62, 2**63 - 1, 2**63, 2**64, 2**100] +INTS = [0, 1, -1, 5, 255, 256, 1000] + BIG + [-x for x in BIG] + +def int_lists() -> List[List[Any]]: + return [ + [], + [1, 2, 3], + [1000, 2000], + [int("1000")], + list(BIG), + [-x for x in BIG], + [1.0, 2.5, NAN], + [True, False], + [IntSub(5), IntSub(1000)], + [IntEqOverride(7)], + [DECIMAL_ONE], + ["1", b"1", None, (1,)], + [None, 2**100, 1000], + ] + +def test_int_in() -> None: + for items in int_lists(): + generic: Any = items + for x in INTS: + expected = x in generic + assert int_in_list(x, items) == expected, (x, items) + assert int_not_in_list(x, items) == (not expected), (x, items) + assert int_in_tuple(x, tuple(items)) == expected, (x, items) + assert int_in_list(1, [AlwaysEqual()]) + assert int_in_tuple(2**100, (AlwaysEqual(),)) + +def test_bool_in() -> None: + t = bool(int() + 1) + for items in int_lists(): + generic: Any = items + assert int_in_list(t, items) == (t in generic), items + assert int_in_list(not t, items) == ((not t) in generic), items + assert int_in_tuple(t, tuple(items)) == (t in generic), items + +STRS = ["", "a", "ab", "abc", "\xe9", "中文", "\U0001f600", "a" * 100] + +def str_lists() -> List[List[Any]]: + return [ + [], + list(STRS), + ["".join(["a", "b"]), "".join(["中", "文"])], + [StrSub("ab"), StrSub("x")], + [StrEqOverride("zzz")], + [b"ab", 1, None], + [None, "abc"], + ] + +def test_str_in() -> None: + for items in str_lists(): + generic: Any = items + for s in STRS + ["x", "".join(["a", "b"])]: + expected = s in generic + assert str_in_list(s, items) == expected, (s, items) + assert str_not_in_list(s, items) == (not expected), (s, items) + assert str_in_tuple(s, tuple(items)) == expected, (s, items) + assert str_in_list("a", [AlwaysEqual()]) + assert str_in_tuple("a", (AlwaysEqual(),)) + +def test_str_subclass_needle() -> None: + s = StrEqOverride("abc") + # The overridden __eq__ of a str subclass is used, like in CPython + assert str_in_list(s, ["x"]) + assert str_in_tuple(s, ("x",)) + assert not str_in_list(s, []) + assert str_in_list(StrSub("abc"), ["abc"]) + assert not str_in_list(StrSub("abc"), ["ab"]) + +def test_subclass_contains_override() -> None: + assert int_in_list(5, ListContainsOverride()) + assert str_in_list("x", ListContainsOverride()) + assert int_in_tuple(5, TupleContainsOverride(())) + assert str_in_tuple("x", TupleContainsOverride(())) + +def test_eq_error() -> None: + with assertRaises(ValueError, "eq"): + int_in_list(1, [RaisingEq()]) + with assertRaises(ValueError, "eq"): + int_in_tuple(2**100, (RaisingEq(),)) + with assertRaises(ValueError, "eq"): + str_in_list("a", [RaisingEq()]) + with assertRaises(ValueError, "eq"): + str_in_tuple("a", (RaisingEq(),)) + # A match before the item with a raising __eq__ is found + assert int_in_list(1, [1, RaisingEq()]) + assert str_in_list("a", ["a", RaisingEq()]) + +def test_list_modified_during_comparison() -> None: + a: List[Any] = [] + a.extend([Shrinker(a), 1, 2, 3]) + assert not int_in_list(3, a) + assert a == [] + b: List[Any] = [] + b.extend([Shrinker(b), "x", "y"]) + assert not str_in_list("y", b) + assert b == [] + +[file contains_helpers.py] +from decimal import Decimal +from typing import Any, List + +NAN = float("nan") +DECIMAL_ONE = Decimal("1") + +class AlwaysEqual: + def __eq__(self, other: object) -> bool: + return True + +class IntSub(int): + pass + +class IntEqOverride(int): + def __eq__(self, other: object) -> bool: + return True + +class StrSub(str): + pass + +class StrEqOverride(str): + def __eq__(self, other: object) -> bool: + return True + +class ListContainsOverride(list): + def __contains__(self, item: object) -> bool: + return True + +class TupleContainsOverride(tuple): + def __contains__(self, item: object) -> bool: + return True + +class RaisingEq: + def __eq__(self, other: object) -> bool: + raise ValueError("eq") + +class Shrinker: + def __init__(self, items: List[Any]) -> None: + self.items = items + + def __eq__(self, other: object) -> bool: + self.items.clear() + return False