diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 2764c7ca182a..cc56c333e312 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1404,16 +1404,25 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RArray: ) -def check_native_int_range(rtype: RPrimitive, n: int) -> bool: - """Is n within the range of a native, fixed-width int type? +def native_int_bounds(rtype: RPrimitive) -> tuple[int, int]: + """Return the smallest and largest values of a native, fixed-width int type. Assume the type is a fixed-width int type. """ if not rtype.is_signed: - return 0 <= n < (1 << (8 * rtype.size)) + return 0, (1 << (8 * rtype.size)) - 1 else: limit = 1 << (rtype.size * 8 - 1) - return -limit <= n < limit + return -limit, limit - 1 + + +def check_native_int_range(rtype: RPrimitive, n: int) -> bool: + """Is n within the range of a native, fixed-width int type? + + Assume the type is a fixed-width int type. + """ + lo, hi = native_int_bounds(rtype) + return lo <= n <= hi def wrap_unsigned_native_int(rtype: RType, n: int) -> int: @@ -1424,7 +1433,8 @@ def wrap_unsigned_native_int(rtype: RType, n: int) -> int: Return n unchanged for other types. """ if is_fixed_width_rtype(rtype) and not rtype.is_signed: - return n & ((1 << (8 * rtype.size)) - 1) + # The largest value of an unsigned type is also a mask of all its bits + return n & native_int_bounds(rtype)[1] return n diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 445a9ee7562e..e17079efa6ce 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -39,11 +39,12 @@ from mypy.patterns import AsPattern, MappingPattern, StarredPattern from mypy.traverser import TraverserVisitor from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types -from mypyc.common import GENERATOR_HELPER_NAME +from mypyc.common import GENERATOR_HELPER_NAME, MAX_SHORT_INT, MIN_SHORT_INT from mypyc.ir.ops import ( ERR_NEVER, BasicBlock, Branch, + ComparisonOp, Integer, IntOp, LoadAddress, @@ -59,6 +60,7 @@ ) from mypyc.ir.rtypes import ( RInstance, + RPrimitive, RTuple, RType, RVec, @@ -76,6 +78,7 @@ is_str_rprimitive, is_tagged, is_tuple_rprimitive, + native_int_bounds, object_pointer_rprimitive, object_rprimitive, short_int_rprimitive, @@ -1466,7 +1469,11 @@ def init(self, start_reg: Value, end_reg: Value, step: int) -> None: self.end_reg = end_reg self.step = step self.end_target = end_reg - if is_short_int_rprimitive(start_reg.type) and is_short_int_rprimitive(end_reg.type): + if ( + is_short_int_rprimitive(start_reg.type) + and is_short_int_rprimitive(end_reg.type) + and not short_int_range_step_may_overflow(end_reg, step) + ): index_type: RType = short_int_rprimitive elif is_fixed_width_rtype(end_reg.type): index_type = end_reg.type @@ -1519,26 +1526,85 @@ def begin_body(self) -> None: def gen_step(self) -> None: builder = self.builder line = self.line + index_type = self.index_reg.type + index = builder.read(self.index_reg, line) # Increment index register. If the range is known to fit in short ints, use - # short ints. - if is_short_int_rprimitive(self.start_reg.type) and is_short_int_rprimitive( - self.end_reg.type - ): + # short ints (init() has checked that the step can't overflow). + if is_short_int_rprimitive(index_type): new_val = builder.int_op( - short_int_rprimitive, - builder.read(self.index_reg, line), - Integer(self.step), - IntOp.ADD, - line, + short_int_rprimitive, index, Integer(self.step), IntOp.ADD, line ) - + elif is_fixed_width_rtype(index_type): + self.gen_fixed_width_overflow_check(index, index_type) + if self.step < 0 and not index_type.is_signed: + # A negative step isn't a valid unsigned value, so subtract its magnitude. + # Signed types add the step, since its magnitude may not fit (-128 for i8). + new_val = builder.binary_op(index, Integer(-self.step), "-", line) + else: + new_val = builder.binary_op(index, Integer(self.step), "+", line) else: - new_val = builder.binary_op( - builder.read(self.index_reg, line), Integer(self.step), "+", line - ) + # Tagged int arithmetic can't overflow. The step may not fit in a short int. + new_val = builder.binary_op(index, builder.load_int(self.step, line), "+", line) builder.assign(self.index_reg, new_val, line) + def gen_fixed_width_overflow_check(self, index: Value, index_type: RPrimitive) -> None: + """Exit the loop if stepping a native int index would overflow. + + If index + step doesn't fit in the index type, it's past the end of the range, + so the loop is done. A step of 1 or -1 can't overflow, since index < end (or + index > end) holds here. + """ + step = self.step + if step in (1, -1): + return + lo, hi = native_int_bounds(index_type) + end = self.end_target + # Overflow if index > hi - step (or index < lo - step, if the step is negative) + limit = (hi if step > 0 else lo) - step + if not lo <= limit <= hi: + # The step doesn't fit in the index type. This is reported as an error elsewhere. + return + if isinstance(end, Integer) and not range_step_may_overflow( + end.numeric_value(), step, lo, hi + ): + return + if step > 0: + op = ComparisonOp.SGT if index_type.is_signed else ComparisonOp.UGT + else: + op = ComparisonOp.SLT if index_type.is_signed else ComparisonOp.ULT + builder = self.builder + line = self.line + overflow = builder.add(ComparisonOp(index, Integer(limit, index_type, line), op, line)) + ok = BasicBlock() + builder.add(Branch(overflow, self.loop_exit, ok, Branch.BOOL, rare=True)) + builder.activate_block(ok) + + +def short_int_range_step_may_overflow(end: Value, step: int) -> bool: + """Could index + step overflow a short int in a short_int range loop? + + Apart from literals, short_int values are lengths, counts and similar values, which + are far from the short int limits. Assume that the magnitude of a non-literal end + value is at most MAX_SHORT_INT // 2. The check is exact for a literal end. + """ + if not MIN_SHORT_INT <= step <= MAX_SHORT_INT: + return True + if isinstance(end, Integer): + end_value = end.numeric_value() + else: + end_value = MAX_SHORT_INT // 2 if step > 0 else -(MAX_SHORT_INT // 2) + return range_step_may_overflow(end_value, step, MIN_SHORT_INT, MAX_SHORT_INT) + + +def range_step_may_overflow(end: int, step: int, lo: int, hi: int) -> bool: + """Could index + step fall outside [lo, hi] for an index in range(..., end, step)? + + The step is only taken when index < end (or index > end, if the step is negative). + """ + last = end - 1 if step > 0 else end + 1 + return not lo <= last + step <= hi + class ForInfiniteCounter(ForGenerator): """Generate optimized IR for a for loop counting from 0 to infinity.""" diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 5555e38c78a7..7e7e9e6d3758 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -127,6 +127,7 @@ is_uint16_rprimitive, is_uint32_rprimitive, is_uint64_rprimitive, + native_int_bounds, none_rprimitive, object_pointer_rprimitive, object_rprimitive, @@ -699,10 +700,8 @@ def check_fixed_width_range( """ in_range, in_range2 = BasicBlock(), BasicBlock() - size = target_type.size - upper_bound = 1 << (size * 8 - 1) - if not target_type.is_signed: - upper_bound *= 2 + lower_bound, max_value = native_int_bounds(target_type) + upper_bound = max_value + 1 # Check if value < upper_bound check_upper = self.add( @@ -713,10 +712,6 @@ def check_fixed_width_range( self.activate_block(in_range) # Check if value >= lower_bound - if target_type.is_signed: - lower_bound = -upper_bound - else: - lower_bound = 0 check_lower = self.add( ComparisonOp(src, Integer(lower_bound, src.type), ComparisonOp.SGE, line) ) diff --git a/mypyc/test-data/irbuild-i16.test b/mypyc/test-data/irbuild-i16.test index a03c9df2c6ac..df2e3dfb0377 100644 --- a/mypyc/test-data/irbuild-i16.test +++ b/mypyc/test-data/irbuild-i16.test @@ -524,3 +524,58 @@ L4: unreachable L5: return r7 + +[case testI16ForLoopOverRangeWithStepAndConstantEnd] +from mypy_extensions import i16 + +def f(a: i16) -> None: + # The largest index is 32764, and 32764 + 3 fits in i16: no overflow check + for x in range(a, i16(32765), 3): + y = x + +def g(a: i16) -> None: + # The largest index is 32765, and 32765 + 3 doesn't fit in i16: overflow check + for x in range(a, i16(32766), 3): + y = x +[out] +def f(a): + a, r0 :: i16 + r1 :: bit + x, y, r2 :: i16 +L0: + r0 = a +L1: + r1 = r0 < 32765 :: signed + if r1 goto L2 else goto L4 :: bool +L2: + x = r0 + y = x +L3: + r2 = r0 + 3 + r0 = r2 + goto L1 +L4: + return 1 +def g(a): + a, r0 :: i16 + r1 :: bit + x, y :: i16 + r2 :: bit + r3 :: i16 +L0: + r0 = a +L1: + r1 = r0 < 32766 :: signed + if r1 goto L2 else goto L5 :: bool +L2: + x = r0 + y = x +L3: + r2 = r0 > 32764 :: signed + if r2 goto L5 else goto L4 :: bool +L4: + r3 = r0 + 3 + r0 = r3 + goto L1 +L5: + return 1 diff --git a/mypyc/test-data/irbuild-i64.test b/mypyc/test-data/irbuild-i64.test index 38c6dd5240ed..821b9765b54f 100644 --- a/mypyc/test-data/irbuild-i64.test +++ b/mypyc/test-data/irbuild-i64.test @@ -1778,6 +1778,64 @@ L3: L4: return 1 +[case testI64ForLoopOverRangeWithStep] +from mypy_extensions import i64 + +def f(a: i64, b: i64) -> None: + for x in range(a, b, 3): + y = x + +def g(a: i64, b: i64) -> None: + for x in range(a, b, -2): + y = x +[out] +def f(a, b): + a, b, r0 :: i64 + r1 :: bit + x, y :: i64 + r2 :: bit + r3 :: i64 +L0: + r0 = a +L1: + r1 = r0 < b :: signed + if r1 goto L2 else goto L5 :: bool +L2: + x = r0 + y = x +L3: + r2 = r0 > 9223372036854775804 :: signed + if r2 goto L5 else goto L4 :: bool +L4: + r3 = r0 + 3 + r0 = r3 + goto L1 +L5: + return 1 +def g(a, b): + a, b, r0 :: i64 + r1 :: bit + x, y :: i64 + r2 :: bit + r3 :: i64 +L0: + r0 = a +L1: + r1 = r0 > b :: signed + if r1 goto L2 else goto L5 :: bool +L2: + x = r0 + y = x +L3: + r2 = r0 < -9223372036854775806 :: signed + if r2 goto L5 else goto L4 :: bool +L4: + r3 = r0 + -2 + r0 = r3 + goto L1 +L5: + return 1 + [case testI64MethodDefaultValueOverride] from mypy_extensions import i64 diff --git a/mypyc/test-data/irbuild-statements.test b/mypyc/test-data/irbuild-statements.test index b2bef7113d00..4146d1f97124 100644 --- a/mypyc/test-data/irbuild-statements.test +++ b/mypyc/test-data/irbuild-statements.test @@ -106,6 +106,56 @@ L3: L4: return 1 +[case testForInRangeStepMayOverflowShortInt_64bit] +def f() -> None: + # 2**62 - 3 + 3 doesn't fit in a short int, so this uses a tagged int index + for i in range(4611686018427387901, 4611686018427387903, 3): + pass +def g(n: list[int]) -> None: + # Still a short int index, since a length can't be close to the limit + for i in range(0, len(n), 3): + pass +[out] +def f(): + r0 :: int + r1 :: bit + i, r2 :: int +L0: + r0 = 9223372036854775802 +L1: + r1 = int_lt r0, 9223372036854775806 + if r1 goto L2 else goto L4 :: bool +L2: + i = r0 +L3: + r2 = CPyTagged_Add(r0, 6) + r0 = r2 + goto L1 +L4: + return 1 +def g(n): + n :: list + r0 :: native_int + r1, r2 :: short_int + r3 :: bit + i :: int + r4 :: short_int +L0: + r0 = var_object_size n + r1 = r0 << 1 + r2 = 0 +L1: + r3 = int_lt r2, r1 + if r3 goto L2 else goto L4 :: bool +L2: + i = r2 +L3: + r4 = r2 + 6 + r2 = r4 + goto L1 +L4: + return 1 + [case testForInRangeNativeIndexArg] class C: def __index__(self) -> int: diff --git a/mypyc/test-data/irbuild-u8.test b/mypyc/test-data/irbuild-u8.test index b5133956c1ba..6867298c57bf 100644 --- a/mypyc/test-data/irbuild-u8.test +++ b/mypyc/test-data/irbuild-u8.test @@ -599,3 +599,83 @@ L4: unreachable L5: return r7 + +[case testU8ForLoopOverRangeWithStep] +from mypy_extensions import u8 + +def f(a: u8, b: u8) -> None: + for x in range(a, b, 5): + y = x + +def g(a: u8, b: u8) -> None: + for x in range(a, b, -5): + y = x + +def h(a: u8, b: u8) -> None: + for x in range(a, b, -1): + y = x +[out] +def f(a, b): + a, b, r0 :: u8 + r1 :: bit + x, y :: u8 + r2 :: bit + r3 :: u8 +L0: + r0 = a +L1: + r1 = r0 < b :: signed + if r1 goto L2 else goto L5 :: bool +L2: + x = r0 + y = x +L3: + r2 = r0 > 250 :: unsigned + if r2 goto L5 else goto L4 :: bool +L4: + r3 = r0 + 5 + r0 = r3 + goto L1 +L5: + return 1 +def g(a, b): + a, b, r0 :: u8 + r1 :: bit + x, y :: u8 + r2 :: bit + r3 :: u8 +L0: + r0 = a +L1: + r1 = r0 > b :: signed + if r1 goto L2 else goto L5 :: bool +L2: + x = r0 + y = x +L3: + r2 = r0 < 5 :: unsigned + if r2 goto L5 else goto L4 :: bool +L4: + r3 = r0 - 5 + r0 = r3 + goto L1 +L5: + return 1 +def h(a, b): + a, b, r0 :: u8 + r1 :: bit + x, y, r2 :: u8 +L0: + r0 = a +L1: + r1 = r0 > b :: signed + if r1 goto L2 else goto L4 :: bool +L2: + x = r0 + y = x +L3: + r2 = r0 - 1 + r0 = r2 + goto L1 +L4: + return 1 diff --git a/mypyc/test-data/run-loops.test b/mypyc/test-data/run-loops.test index b8d3e5ceef32..59a3392b0d48 100644 --- a/mypyc/test-data/run-loops.test +++ b/mypyc/test-data/run-loops.test @@ -3,7 +3,7 @@ [case testForRange] from typing import Any -from mypy_extensions import i64 +from mypy_extensions import i64, i16, u8, u64 from testutil import assertRaises def count(n: int) -> list[int]: @@ -46,6 +46,145 @@ def test_range_loop() -> None: assert count_double(10, 15) == [10, 12, 14] assert count_down_short() == list(range(10, 0, -1)) +# The range loops below stop after 10 items, so that a wrapped-around index +# makes the test fail instead of hang. + +def range_up_i64(a: i64, b: i64) -> list[int]: + result = [] + for i in range(a, b, 3): + result.append(int(i)) + if len(result) > 10: + break + return result + +def range_down_i64(a: i64, b: i64) -> list[int]: + result = [] + for i in range(a, b, -3): + result.append(int(i)) + if len(result) > 10: + break + return result + +def range_up_i16(a: i16, b: i16) -> list[int]: + result = [] + for i in range(a, b, 4): + result.append(int(i)) + if len(result) > 10: + break + return result + +def range_down_i16(a: i16, b: i16) -> list[int]: + result = [] + for i in range(a, b, -4): + result.append(int(i)) + if len(result) > 10: + break + return result + +def range_up_u8(a: u8, b: u8) -> list[int]: + result = [] + for i in range(a, b, 5): + result.append(int(i)) + if len(result) > 10: + break + return result + +def range_down_u8(a: u8, b: u8) -> list[int]: + result = [] + for i in range(a, b, -5): + result.append(int(i)) + if len(result) > 10: + break + return result + +def range_down_u64_by_one(a: u64, b: u64) -> list[int]: + result = [] + for i in range(a, b, -1): + result.append(int(i)) + return result + +def range_up_i16_constant_end(a: i16) -> list[int]: + result = [] + for i in range(a, i16(32766), 3): + result.append(int(i)) + if len(result) > 10: + break + return result + +def range_up_i16_constant_end_no_check(a: i16) -> list[int]: + result = [] + for i in range(a, i16(32765), 3): + result.append(int(i)) + if len(result) > 10: + break + return result + +def test_range_step_near_native_int_limits() -> None: + for d1 in range(8): + for d2 in range(4): + a, b = 2**63 - 1 - d1, 2**63 - 1 - d2 + assert range_up_i64(a, b) == list(range(a, b, 3)), (a, b) + a, b = -2**63 + d1, -2**63 + d2 + assert range_down_i64(a, b) == list(range(a, b, -3)), (a, b) + a, b = 2**15 - 1 - d1, 2**15 - 1 - d2 + assert range_up_i16(a, b) == list(range(a, b, 4)), (a, b) + a, b = -2**15 + d1, -2**15 + d2 + assert range_down_i16(a, b) == list(range(a, b, -4)), (a, b) + assert range_up_i64(0, 9) == [0, 3, 6] + assert range_down_i64(9, 0) == [9, 6, 3] + for d1 in range(12): + for d2 in range(4): + a = 255 - d1 + b = 255 - d2 + assert range_up_u8(a, b) == list(range(a, b, 5)), (a, b) + assert range_down_u8(d1, d2) == list(range(d1, d2, -5)), (d1, d2) + assert range_down_u8(255, 240) == [255, 250, 245] + assert range_down_u64_by_one(3, 0) == [3, 2, 1] + assert range_down_u64_by_one(2**64 - 1, 2**64 - 3) == [2**64 - 1, 2**64 - 2] + for a in range(32755, 32767): + assert range_up_i16_constant_end(a) == list(range(a, 32766, 3)), a + assert range_up_i16_constant_end_no_check(a) == list(range(a, 32765, 3)), a + +def range_up_short_literal() -> list[int]: + result = [] + for i in range(4611686018427387897, 4611686018427387903, 3): # 2**62 - 7, 2**62 - 1 + result.append(i) + if len(result) > 10: + break + return result + +def range_down_short_literal() -> list[int]: + result = [] + for i in range(-4611686018427387898, -4611686018427387904, -3): # -2**62 + 6, -2**62 + result.append(i) + if len(result) > 10: + break + return result + +def range_huge_step_literal() -> list[int]: + result = [] + for i in range(-5, 10, 4611686018427387904): # step 2**62 + result.append(i) + if len(result) > 10: + break + return result + +def range_huge_step(a: int, b: int) -> list[int]: + result = [] + for i in range(a, b, 4611686018427387904): # step 2**62 + result.append(i) + if len(result) > 10: + break + return result + +def test_range_step_near_short_int_limits() -> None: + n = 2**62 + assert range_up_short_literal() == list(range(n - 7, n - 1, 3)) + assert range_down_short_literal() == list(range(-n + 6, -n, -3)) + assert range_huge_step_literal() == [-5] + assert range_huge_step(0, 10) == [0] + assert range_huge_step(-n, n) == [-n, 0] + def sum_range(x: range) -> int: sum = 0 for i in x: