diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index d33f15a68bee..af51989b28fa 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -1823,9 +1823,11 @@ def translate_bytes_writer_set_item( # Get the BytesWriter object obj = builder.accept(base_expr) - # Get the index and value arguments - index = builder.accept(args[0]) - value = builder.accept(args[1]) + # Get the index and value arguments. Convert them before checking the index, + # since the conversions can call __index__, which can change the length or the + # buffer of the writer. + index = builder.coerce(builder.accept(args[0]), int64_rprimitive, ctx_expr.line) + value = builder.coerce(builder.accept(args[1]), uint8_rprimitive, ctx_expr.line) # Adjust the index (handle negative indices) adjusted_index = builder.primitive_op( diff --git a/mypyc/lib-rt/byteswriter_extra_ops.c b/mypyc/lib-rt/byteswriter_extra_ops.c index c94fe80502d6..f7d4105bf79a 100644 --- a/mypyc/lib-rt/byteswriter_extra_ops.c +++ b/mypyc/lib-rt/byteswriter_extra_ops.c @@ -3,18 +3,7 @@ #include "byteswriter_extra_ops.h" -char CPyBytesWriter_Write(PyObject *obj, PyObject *value) { - BytesWriterObject *self = (BytesWriterObject *)obj; - const char *data; - Py_ssize_t size; - if (likely(PyBytes_Check(value))) { - data = PyBytes_AS_STRING(value); - size = PyBytes_GET_SIZE(value); - } else { - data = PyByteArray_AS_STRING(value); - size = PyByteArray_GET_SIZE(value); - } - // Write bytes content. +static inline char write_data(BytesWriterObject *self, const char *data, Py_ssize_t size) { if (!CPyBytesWriter_EnsureSize(self, size)) return CPY_NONE_ERROR; if (size < 8) { @@ -30,6 +19,20 @@ char CPyBytesWriter_Write(PyObject *obj, PyObject *value) { return CPY_NONE; } +char CPyBytesWriter_Write(PyObject *obj, PyObject *value) { + BytesWriterObject *self = (BytesWriterObject *)obj; + if (likely(PyBytes_Check(value))) { + return write_data(self, PyBytes_AS_STRING(value), PyBytes_GET_SIZE(value)); + } + // On free-threaded builds another thread could resize the bytearray while we + // copy it, so lock it like the bytearray methods do (no-op on other builds). + char result; + Py_BEGIN_CRITICAL_SECTION(value); + result = write_data(self, PyByteArray_AS_STRING(value), PyByteArray_GET_SIZE(value)); + Py_END_CRITICAL_SECTION(); + return result; +} + void CPyBytes_ReadError(int64_t index, Py_ssize_t size) { if (index < 0) { PyErr_SetString(PyExc_ValueError, "index must be non-negative"); diff --git a/mypyc/lib-rt/strings/librt_strings.c b/mypyc/lib-rt/strings/librt_strings.c index ef0df125faa1..5d04b3bec3bd 100644 --- a/mypyc/lib-rt/strings/librt_strings.c +++ b/mypyc/lib-rt/strings/librt_strings.c @@ -177,27 +177,26 @@ BytesWriter_item(BytesWriterObject *self, Py_ssize_t index) static int BytesWriter_ass_item(BytesWriterObject *self, Py_ssize_t index, PyObject *value) { - Py_ssize_t length = self->len; - - // Check bounds - if (index < 0 || index >= length) { - PyErr_SetString(PyExc_IndexError, "BytesWriter index out of range"); - return -1; - } - // Check that value is not NULL (deletion not supported) if (value == NULL) { PyErr_SetString(PyExc_TypeError, "BytesWriter does not support item deletion"); return -1; } - // Convert value to uint8 + // Convert value to uint8. Do this before checking bounds, since this can call + // __index__, which can change the length or the buffer of the writer. uint8_t byte_value = CPyLong_AsUInt8(value); if (unlikely(byte_value == CPY_LL_UINT_ERROR && PyErr_Occurred())) { CPy_TypeError("u8", value); return -1; } + // Check bounds + if (index < 0 || index >= self->len) { + PyErr_SetString(PyExc_IndexError, "BytesWriter index out of range"); + return -1; + } + // Assign the byte self->buf[index] = (char)byte_value; return 0; @@ -256,17 +255,7 @@ check_bytes_writer(PyObject *data) { } static char -BytesWriter_write_internal(BytesWriterObject *self, PyObject *value) { - const char *data; - Py_ssize_t size; - if (likely(PyBytes_Check(value))) { - data = PyBytes_AS_STRING(value); - size = PyBytes_GET_SIZE(value); - } else { - data = PyByteArray_AS_STRING(value); - size = PyByteArray_GET_SIZE(value); - } - // Write bytes content. +BytesWriter_write_data(BytesWriterObject *self, const char *data, Py_ssize_t size) { if (!ensure_bytes_writer_size(self, size)) return CPY_NONE_ERROR; memcpy(self->buf + self->len, data, size); @@ -274,6 +263,20 @@ BytesWriter_write_internal(BytesWriterObject *self, PyObject *value) { return CPY_NONE; } +static char +BytesWriter_write_internal(BytesWriterObject *self, PyObject *value) { + if (likely(PyBytes_Check(value))) { + return BytesWriter_write_data(self, PyBytes_AS_STRING(value), PyBytes_GET_SIZE(value)); + } + // On free-threaded builds another thread could resize the bytearray while we + // copy it, so lock it like the bytearray methods do (no-op on other builds). + char result; + Py_BEGIN_CRITICAL_SECTION(value); + result = BytesWriter_write_data(self, PyByteArray_AS_STRING(value), PyByteArray_GET_SIZE(value)); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject* BytesWriter_write(PyObject *self, PyObject *const *args, size_t nargs) { if (unlikely(nargs != 1)) { diff --git a/mypyc/test-data/run-librt-strings.test b/mypyc/test-data/run-librt-strings.test index e48f50601140..308df6fdd552 100644 --- a/mypyc/test-data/run-librt-strings.test +++ b/mypyc/test-data/run-librt-strings.test @@ -1813,3 +1813,63 @@ def test_codepoint_case_conversion_via_any() -> None: f(1 << 40) with assertRaises(OverflowError, "codepoint out of i32 range"): f(-(1 << 40)) + +[case testLibrtBytesWriterSetItemValueResetsWriter_librt] +from typing import Any + +from librt.strings import BytesWriter + +from testutil import assertRaises +from bytes_writer_helper import ByteArraySubclass, ResetWriter + +def set_item(w: BytesWriter, i: int, v: Any) -> None: + w[i] = v + +def test_compiled_set_item() -> None: + # Converting the value calls __index__, which resets the writer to its + # small initial buffer. The index must be checked after that. + for n in (300, 100000): + w = BytesWriter() + w.write(b"x" * n) + with assertRaises(IndexError): + set_item(w, n - 1, ResetWriter(w)) + assert len(w) == 0 + +def test_generic_set_item() -> None: + for n in (300, 100000): + w = BytesWriter() + w.write(b"x" * n) + a: Any = w + with assertRaises(IndexError): + a[n - 1] = ResetWriter(w) + assert len(w) == 0 + +def test_value_checked_before_index() -> None: + w = BytesWriter() + w.write(b"x") + a: Any = w + # Like bytearray, an invalid value is reported before an invalid index + with assertRaises(TypeError): + a[5] = 256 + with assertRaises(TypeError): + a[5] = "x" + +def test_write_bytearray_subclass() -> None: + w = BytesWriter() + w.write(ByteArraySubclass(b"abc")) + w.write(bytearray(b"d" * 100)) + assert w.getvalue() == b"abc" + b"d" * 100 + +[file bytes_writer_helper.py] +from typing import Any + +class ResetWriter: + def __init__(self, w: Any) -> None: + self.w = w + + def __index__(self) -> int: + self.w.__init__() + return 65 + +class ByteArraySubclass(bytearray): + pass