From 00bac09f97f44d2757cb1416abc8684e2eb89981 Mon Sep 17 00:00:00 2001 From: Jahongir Qurbonov Date: Sat, 14 Nov 2026 13:47:11 +0500 Subject: [PATCH] [mypyc] Fix memory safety bugs in librt.strings.BytesWriter - BytesWriter.__setitem__ checked the index before converting the value. The conversion can call __index__, which can reset the writer to its small initial buffer, so the byte was written out of bounds. Convert the value (and the index) before the bounds check, in both the compiled and the generic path. Like bytearray, an invalid value is now reported before an invalid index. - BytesWriter.write(bytearray) read the buffer and size of the bytearray without locking it. On free-threaded builds another thread could resize the bytearray during the copy. Lock it with a critical section, like the bytearray methods do. This is a no-op on builds with the GIL. --- mypyc/irbuild/specialize.py | 8 ++-- mypyc/lib-rt/byteswriter_extra_ops.c | 27 ++++++------ mypyc/lib-rt/strings/librt_strings.c | 43 +++++++++--------- mypyc/test-data/run-librt-strings.test | 60 ++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 35 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index d33f15a68beec..af51989b28fa3 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -1823,9 +1823,11 @@ def translate_bytes_writer_set_item( # Get the BytesWriter object obj = builder.accept(base_expr) - # Get the index and value arguments - index = builder.accept(args[0]) - value = builder.accept(args[1]) + # Get the index and value arguments. Convert them before checking the index, + # since the conversions can call __index__, which can change the length or the + # buffer of the writer. + index = builder.coerce(builder.accept(args[0]), int64_rprimitive, ctx_expr.line) + value = builder.coerce(builder.accept(args[1]), uint8_rprimitive, ctx_expr.line) # Adjust the index (handle negative indices) adjusted_index = builder.primitive_op( diff --git a/mypyc/lib-rt/byteswriter_extra_ops.c b/mypyc/lib-rt/byteswriter_extra_ops.c index c94fe80502d6c..f7d4105bf79a4 100644 --- a/mypyc/lib-rt/byteswriter_extra_ops.c +++ b/mypyc/lib-rt/byteswriter_extra_ops.c @@ -3,18 +3,7 @@ #include "byteswriter_extra_ops.h" -char CPyBytesWriter_Write(PyObject *obj, PyObject *value) { - BytesWriterObject *self = (BytesWriterObject *)obj; - const char *data; - Py_ssize_t size; - if (likely(PyBytes_Check(value))) { - data = PyBytes_AS_STRING(value); - size = PyBytes_GET_SIZE(value); - } else { - data = PyByteArray_AS_STRING(value); - size = PyByteArray_GET_SIZE(value); - } - // Write bytes content. +static inline char write_data(BytesWriterObject *self, const char *data, Py_ssize_t size) { if (!CPyBytesWriter_EnsureSize(self, size)) return CPY_NONE_ERROR; if (size < 8) { @@ -30,6 +19,20 @@ char CPyBytesWriter_Write(PyObject *obj, PyObject *value) { return CPY_NONE; } +char CPyBytesWriter_Write(PyObject *obj, PyObject *value) { + BytesWriterObject *self = (BytesWriterObject *)obj; + if (likely(PyBytes_Check(value))) { + return write_data(self, PyBytes_AS_STRING(value), PyBytes_GET_SIZE(value)); + } + // On free-threaded builds another thread could resize the bytearray while we + // copy it, so lock it like the bytearray methods do (no-op on other builds). + char result; + Py_BEGIN_CRITICAL_SECTION(value); + result = write_data(self, PyByteArray_AS_STRING(value), PyByteArray_GET_SIZE(value)); + Py_END_CRITICAL_SECTION(); + return result; +} + void CPyBytes_ReadError(int64_t index, Py_ssize_t size) { if (index < 0) { PyErr_SetString(PyExc_ValueError, "index must be non-negative"); diff --git a/mypyc/lib-rt/strings/librt_strings.c b/mypyc/lib-rt/strings/librt_strings.c index ef0df125faa11..5d04b3bec3bd9 100644 --- a/mypyc/lib-rt/strings/librt_strings.c +++ b/mypyc/lib-rt/strings/librt_strings.c @@ -177,27 +177,26 @@ BytesWriter_item(BytesWriterObject *self, Py_ssize_t index) static int BytesWriter_ass_item(BytesWriterObject *self, Py_ssize_t index, PyObject *value) { - Py_ssize_t length = self->len; - - // Check bounds - if (index < 0 || index >= length) { - PyErr_SetString(PyExc_IndexError, "BytesWriter index out of range"); - return -1; - } - // Check that value is not NULL (deletion not supported) if (value == NULL) { PyErr_SetString(PyExc_TypeError, "BytesWriter does not support item deletion"); return -1; } - // Convert value to uint8 + // Convert value to uint8. Do this before checking bounds, since this can call + // __index__, which can change the length or the buffer of the writer. uint8_t byte_value = CPyLong_AsUInt8(value); if (unlikely(byte_value == CPY_LL_UINT_ERROR && PyErr_Occurred())) { CPy_TypeError("u8", value); return -1; } + // Check bounds + if (index < 0 || index >= self->len) { + PyErr_SetString(PyExc_IndexError, "BytesWriter index out of range"); + return -1; + } + // Assign the byte self->buf[index] = (char)byte_value; return 0; @@ -256,17 +255,7 @@ check_bytes_writer(PyObject *data) { } static char -BytesWriter_write_internal(BytesWriterObject *self, PyObject *value) { - const char *data; - Py_ssize_t size; - if (likely(PyBytes_Check(value))) { - data = PyBytes_AS_STRING(value); - size = PyBytes_GET_SIZE(value); - } else { - data = PyByteArray_AS_STRING(value); - size = PyByteArray_GET_SIZE(value); - } - // Write bytes content. +BytesWriter_write_data(BytesWriterObject *self, const char *data, Py_ssize_t size) { if (!ensure_bytes_writer_size(self, size)) return CPY_NONE_ERROR; memcpy(self->buf + self->len, data, size); @@ -274,6 +263,20 @@ BytesWriter_write_internal(BytesWriterObject *self, PyObject *value) { return CPY_NONE; } +static char +BytesWriter_write_internal(BytesWriterObject *self, PyObject *value) { + if (likely(PyBytes_Check(value))) { + return BytesWriter_write_data(self, PyBytes_AS_STRING(value), PyBytes_GET_SIZE(value)); + } + // On free-threaded builds another thread could resize the bytearray while we + // copy it, so lock it like the bytearray methods do (no-op on other builds). + char result; + Py_BEGIN_CRITICAL_SECTION(value); + result = BytesWriter_write_data(self, PyByteArray_AS_STRING(value), PyByteArray_GET_SIZE(value)); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject* BytesWriter_write(PyObject *self, PyObject *const *args, size_t nargs) { if (unlikely(nargs != 1)) { diff --git a/mypyc/test-data/run-librt-strings.test b/mypyc/test-data/run-librt-strings.test index e48f506011409..308df6fdd552b 100644 --- a/mypyc/test-data/run-librt-strings.test +++ b/mypyc/test-data/run-librt-strings.test @@ -1813,3 +1813,63 @@ def test_codepoint_case_conversion_via_any() -> None: f(1 << 40) with assertRaises(OverflowError, "codepoint out of i32 range"): f(-(1 << 40)) + +[case testLibrtBytesWriterSetItemValueResetsWriter_librt] +from typing import Any + +from librt.strings import BytesWriter + +from testutil import assertRaises +from bytes_writer_helper import ByteArraySubclass, ResetWriter + +def set_item(w: BytesWriter, i: int, v: Any) -> None: + w[i] = v + +def test_compiled_set_item() -> None: + # Converting the value calls __index__, which resets the writer to its + # small initial buffer. The index must be checked after that. + for n in (300, 100000): + w = BytesWriter() + w.write(b"x" * n) + with assertRaises(IndexError): + set_item(w, n - 1, ResetWriter(w)) + assert len(w) == 0 + +def test_generic_set_item() -> None: + for n in (300, 100000): + w = BytesWriter() + w.write(b"x" * n) + a: Any = w + with assertRaises(IndexError): + a[n - 1] = ResetWriter(w) + assert len(w) == 0 + +def test_value_checked_before_index() -> None: + w = BytesWriter() + w.write(b"x") + a: Any = w + # Like bytearray, an invalid value is reported before an invalid index + with assertRaises(TypeError): + a[5] = 256 + with assertRaises(TypeError): + a[5] = "x" + +def test_write_bytearray_subclass() -> None: + w = BytesWriter() + w.write(ByteArraySubclass(b"abc")) + w.write(bytearray(b"d" * 100)) + assert w.getvalue() == b"abc" + b"d" * 100 + +[file bytes_writer_helper.py] +from typing import Any + +class ResetWriter: + def __init__(self, w: Any) -> None: + self.w = w + + def __index__(self) -> int: + self.w.__init__() + return 65 + +class ByteArraySubclass(bytearray): + pass