From 4b355b40920d9cc7d4cd9b1891817f7c308f52f5 Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Wed, 29 Jul 2026 11:50:38 +0300 Subject: [PATCH 1/3] gh-154859: Keep the iconv shift state across incremental decode calls The iconv codecs opened a fresh conversion for every call, so decoding a stateful encoding in chunks lost the shift state and silently produced wrong text: incremental decoding of ISO-2022-CN dropped the escape sequences and returned the raw bytes as ASCII. _codecs.iconv_state() now opens a conversion that the incremental decoder and the stream reader keep and pass back to iconv_decode(), so one conversion spans the whole stream. reset() starts a new one. --- Include/internal/pycore_unicodeobject.h | 9 +++- Lib/encodings/_iconv_codecs.py | 22 ++++++-- Lib/test/test_codecs.py | 24 +++++++++ Modules/_codecsmodule.c | 72 +++++++++++++++++++++++-- Modules/clinic/_codecsmodule.c.h | 65 ++++++++++++++++++++-- Objects/unicodeobject.c | 32 ++++++++--- 6 files changed, 206 insertions(+), 18 deletions(-) diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 012f5da2869cd5..098818e9f6d1b5 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -185,12 +185,19 @@ extern int _PyUnicodeWriter_FormatV( /* --- iconv Codec -------------------------------------------------------- */ #ifdef HAVE_ICONV +#include + +/* Open a conversion for decoding ENCODING, to reuse across calls. Returns + (iconv_t)-1 with an exception set on failure. */ +extern iconv_t _PyUnicode_IconvOpenDecoder(const char *encoding); + extern PyObject* _PyUnicode_DecodeIconv( const char *encoding, /* iconv encoding name */ const char *string, /* encoded string */ Py_ssize_t length, /* size of string */ const char *errors, /* error handling */ - Py_ssize_t *consumed); /* bytes consumed, or NULL for non-stateful */ + Py_ssize_t *consumed, /* bytes consumed, or NULL for non-stateful */ + iconv_t *cdp); /* conversion to reuse, or NULL to open one */ extern PyObject* _PyUnicode_EncodeIconv( const char *encoding, /* iconv encoding name */ diff --git a/Lib/encodings/_iconv_codecs.py b/Lib/encodings/_iconv_codecs.py index 1c701e0af423af..51c5f77e87fc46 100644 --- a/Lib/encodings/_iconv_codecs.py +++ b/Lib/encodings/_iconv_codecs.py @@ -1,7 +1,7 @@ import codecs def create_iconv_codec(name, encoding): - from _codecs import iconv_encode, iconv_decode + from _codecs import iconv_encode, iconv_decode, iconv_state def encode(input, errors='strict'): return iconv_encode(encoding, input, errors) @@ -14,16 +14,32 @@ def encode(self, input, final=False): return iconv_encode(encoding, input, self.errors)[0] class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def __init__(self, errors='strict'): + super().__init__(errors) + self._state = iconv_state(encoding) + def _buffer_decode(self, input, errors, final): - return iconv_decode(encoding, input, errors, final) + return iconv_decode(encoding, input, errors, final, self._state) + + def reset(self): + super().reset() + self._state = iconv_state(encoding) class StreamWriter(codecs.StreamWriter): def encode(self, input, errors='strict'): return iconv_encode(encoding, input, errors) class StreamReader(codecs.StreamReader): + def __init__(self, stream, errors='strict'): + super().__init__(stream, errors) + self._state = iconv_state(encoding) + def decode(self, input, errors, final=False): - return iconv_decode(encoding, input, errors, final) + return iconv_decode(encoding, input, errors, final, self._state) + + def reset(self): + super().reset() + self._state = iconv_state(encoding) return codecs.CodecInfo( name=name, diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 31704955df3e14..50b9689ef291d8 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3676,6 +3676,10 @@ def iconv_encoding_available(name): ('ISO-8859-1', 'Grüße'), ] _ICONV_MULTIBYTE = ['EUC-JP', 'SHIFT_JIS', 'GBK', 'GB18030', 'BIG5'] +# Stateful encodings: the shift state set by an escape sequence has to survive +# from one incremental call to the next. CPython has no built-in codec for +# these, so the plain name reaches the iconv codec. +_ICONV_STATEFUL = ['ISO-2022-CN'] # Encodings iconv may provide but for which CPython has no built-in codec # (cp1047 is EBCDIC, i.e. not ASCII-compatible). _ICONV_ONLY = ['cp1047', 'cp1133', 'GEORGIAN-PS', 'ARMSCII-8'] @@ -3804,6 +3808,26 @@ def test_stream(self): reader = codecs.getreader('iconv:' + enc)(io.BytesIO(raw)) self.assertEqual(reader.read(), text) + def test_incremental_decode_shift_state(self): + enc = self.require(*_ICONV_STATEFUL) + text = 'ABC\u4e2d\u6587DEF' + data = codecs.encode(text, 'iconv:' + enc) + self.assertEqual(codecs.decode(data, 'iconv:' + enc), text) + dec = codecs.getincrementaldecoder('iconv:' + enc)() + out = ''.join(dec.decode(data[i:i+1]) for i in range(len(data))) + out += dec.decode(b'', True) + self.assertEqual(out, text) + # reset() starts a new conversion, so decoding can begin again. + dec.reset() + self.assertEqual(dec.decode(data, True), text) + + def test_stream_shift_state(self): + enc = self.require(*_ICONV_STATEFUL) + text = 'ABC\u4e2d\u6587DEF' + raw = codecs.encode(text, 'iconv:' + enc) + reader = codecs.getreader('iconv:' + enc)(io.BytesIO(raw)) + self.assertEqual(''.join(iter(lambda: reader.read(1), '')), text) + def test_encode_kinds(self): # The string's own buffer is fed to iconv per storage kind; check each # of the 1-, 2- and 4-byte kinds against the built-in codec. diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 7cba234fc80b59..cffc0295ad0d3d 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -644,24 +644,89 @@ _codecs_code_page_decode_impl(PyObject *module, int codepage, #ifdef HAVE_ICONV +#ifdef HAVE_ICONV +#define ICONV_STATE_CAPSULE "_codecs.iconv_state" + +static void +iconv_state_destructor(PyObject *capsule) +{ + iconv_t *cdp = PyCapsule_GetPointer(capsule, ICONV_STATE_CAPSULE); + if (cdp == NULL) { + PyErr_Clear(); + return; + } + iconv_close(*cdp); + PyMem_Free(cdp); +} +#endif + +/*[clinic input] +_codecs.iconv_state + + encoding: str + / + +Open an iconv conversion for decoding, to reuse across calls. + +The result is an opaque object. Reusing one conversion keeps the +shift state of a stateful encoding, such as ISO-2022-CN, across calls. +[clinic start generated code]*/ + +static PyObject * +_codecs_iconv_state_impl(PyObject *module, const char *encoding) +/*[clinic end generated code: output=4100a9b65a64d65c input=6ffbfa5bb1d2d208]*/ +{ +#ifdef HAVE_ICONV + iconv_t *cdp = PyMem_Malloc(sizeof(iconv_t)); + if (cdp == NULL) { + return PyErr_NoMemory(); + } + *cdp = _PyUnicode_IconvOpenDecoder(encoding); + if (*cdp == (iconv_t)-1) { + PyMem_Free(cdp); + return NULL; + } + PyObject *capsule = PyCapsule_New(cdp, ICONV_STATE_CAPSULE, + iconv_state_destructor); + if (capsule == NULL) { + iconv_close(*cdp); + PyMem_Free(cdp); + return NULL; + } + return capsule; +#else + PyErr_SetString(PyExc_LookupError, "iconv is not available"); + return NULL; +#endif +} + /*[clinic input] _codecs.iconv_decode encoding: str data: Py_buffer errors: str(accept={str, NoneType}) = None final: bool = False + state: object = None / [clinic start generated code]*/ static PyObject * _codecs_iconv_decode_impl(PyObject *module, const char *encoding, - Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=6c6145a9decc2ba8 input=d15a04d7d3a3e0cd]*/ + Py_buffer *data, const char *errors, int final, + PyObject *state) +/*[clinic end generated code: output=ed99087a9b21d007 input=b23f4298c963f9c5]*/ { + iconv_t *cdp = NULL; + if (state != Py_None) { + cdp = PyCapsule_GetPointer(state, ICONV_STATE_CAPSULE); + if (cdp == NULL) { + return NULL; + } + } Py_ssize_t consumed = data->len; PyObject *decoded = _PyUnicode_DecodeIconv(encoding, data->buf, data->len, errors, - final ? NULL : &consumed); + final ? NULL : &consumed, cdp); return codec_tuple(decoded, consumed); } @@ -1155,6 +1220,7 @@ static PyMethodDef _codecs_functions[] = { _CODECS_CODE_PAGE_DECODE_METHODDEF _CODECS_ICONV_ENCODE_METHODDEF _CODECS_ICONV_DECODE_METHODDEF + _CODECS_ICONV_STATE_METHODDEF _CODECS_REGISTER_ERROR_METHODDEF _CODECS__UNREGISTER_ERROR_METHODDEF _CODECS_LOOKUP_ERROR_METHODDEF diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 788b9b706c8fbb..4507f61ab2c287 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -1631,8 +1631,53 @@ _codecs_code_page_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar #if defined(HAVE_ICONV) +PyDoc_STRVAR(_codecs_iconv_state__doc__, +"iconv_state($module, encoding, /)\n" +"--\n" +"\n" +"Open an iconv conversion for decoding, to reuse across calls.\n" +"\n" +"The result is an opaque object. Reusing one conversion keeps the\n" +"shift state of a stateful encoding, such as ISO-2022-CN, across calls."); + +#define _CODECS_ICONV_STATE_METHODDEF \ + {"iconv_state", (PyCFunction)_codecs_iconv_state, METH_O, _codecs_iconv_state__doc__}, + +static PyObject * +_codecs_iconv_state_impl(PyObject *module, const char *encoding); + +static PyObject * +_codecs_iconv_state(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *encoding; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("iconv_state", "argument", "str", arg); + goto exit; + } + Py_ssize_t encoding_length; + encoding = PyUnicode_AsUTF8AndSize(arg, &encoding_length); + if (encoding == NULL) { + goto exit; + } + if (strlen(encoding) != (size_t)encoding_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = _codecs_iconv_state_impl(module, encoding); + +exit: + return return_value; +} + +#endif /* defined(HAVE_ICONV) */ + +#if defined(HAVE_ICONV) + PyDoc_STRVAR(_codecs_iconv_decode__doc__, -"iconv_decode($module, encoding, data, errors=None, final=False, /)\n" +"iconv_decode($module, encoding, data, errors=None, final=False,\n" +" state=None, /)\n" "--\n" "\n"); @@ -1641,7 +1686,8 @@ PyDoc_STRVAR(_codecs_iconv_decode__doc__, static PyObject * _codecs_iconv_decode_impl(PyObject *module, const char *encoding, - Py_buffer *data, const char *errors, int final); + Py_buffer *data, const char *errors, int final, + PyObject *state); static PyObject * _codecs_iconv_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -1651,8 +1697,9 @@ _codecs_iconv_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_buffer data = {NULL, NULL}; const char *errors = NULL; int final = 0; + PyObject *state = Py_None; - if (!_PyArg_CheckPositional("iconv_decode", nargs, 2, 4)) { + if (!_PyArg_CheckPositional("iconv_decode", nargs, 2, 5)) { goto exit; } if (!PyUnicode_Check(args[0])) { @@ -1699,8 +1746,12 @@ _codecs_iconv_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (final < 0) { goto exit; } + if (nargs < 5) { + goto skip_optional; + } + state = args[4]; skip_optional: - return_value = _codecs_iconv_decode_impl(module, encoding, &data, errors, final); + return_value = _codecs_iconv_decode_impl(module, encoding, &data, errors, final, state); exit: /* Cleanup for data */ @@ -3014,6 +3065,10 @@ _codecs__normalize_encoding(PyObject *module, PyObject *const *args, Py_ssize_t #define _CODECS_CODE_PAGE_DECODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_DECODE_METHODDEF) */ +#ifndef _CODECS_ICONV_STATE_METHODDEF + #define _CODECS_ICONV_STATE_METHODDEF +#endif /* !defined(_CODECS_ICONV_STATE_METHODDEF) */ + #ifndef _CODECS_ICONV_DECODE_METHODDEF #define _CODECS_ICONV_DECODE_METHODDEF #endif /* !defined(_CODECS_ICONV_DECODE_METHODDEF) */ @@ -3033,4 +3088,4 @@ _codecs__normalize_encoding(PyObject *module, PyObject *const *args, Py_ssize_t #ifndef _CODECS_ICONV_ENCODE_METHODDEF #define _CODECS_ICONV_ENCODE_METHODDEF #endif /* !defined(_CODECS_ICONV_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=912e04020d6a6144 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=30b2a2c3eb23dfc1 input=a9049054013a1b77]*/ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a4550c0f5f3363..888c289ce24ff9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -8239,26 +8239,42 @@ iconv_open_or_set_error(const char *tocode, const char *fromcode, return cd; } +iconv_t +_PyUnicode_IconvOpenDecoder(const char *encoding) +{ + return iconv_open_or_set_error(ICONV_PIVOT, encoding, encoding); +} + /* * Decode bytes with iconv() into a str. * * The input is converted to native-endian UTF-32 one chunk at a time and * appended to a _PyUnicodeWriter. If *consumed* is non-NULL the decode is * stateful: a trailing incomplete sequence stops and sets *consumed*. + * + * If *cdp* is non-NULL the conversion it points to is used and left open, so + * that a shift state survives from one call to the next. */ PyObject * _PyUnicode_DecodeIconv(const char *encoding, const char *s, Py_ssize_t size, - const char *errors, Py_ssize_t *consumed) + const char *errors, Py_ssize_t *consumed, + iconv_t *cdp) { if (size < 0) { PyErr_BadInternalCall(); return NULL; } - iconv_t cd = iconv_open_or_set_error(ICONV_PIVOT, encoding, encoding); - if (cd == (iconv_t)-1) { - return NULL; + iconv_t cd; + if (cdp != NULL) { + cd = *cdp; + } + else { + cd = _PyUnicode_IconvOpenDecoder(encoding); + if (cd == (iconv_t)-1) { + return NULL; + } } /* Scratch buffer for one iconv() output chunk, as UTF-32 code points. */ @@ -8337,13 +8353,17 @@ _PyUnicode_DecodeIconv(const char *encoding, if (consumed != NULL) { *consumed = in - starts; } - iconv_close(cd); + if (cdp == NULL) { + iconv_close(cd); + } Py_XDECREF(errorHandler); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); error: - iconv_close(cd); + if (cdp == NULL) { + iconv_close(cd); + } _PyUnicodeWriter_Dealloc(&writer); Py_XDECREF(errorHandler); Py_XDECREF(exc); From 386b0d80551d321ed81c267df408082c8d71167e Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Wed, 5 Aug 2026 22:39:44 +0300 Subject: [PATCH 2/3] Do not build the test input with the platform iconv An iconv that provides ISO-2022-CN may still be unable to encode Chinese text, as macOS and iOS cannot, and the encode raised before the decoding under test ran. The bytes are now fixed in the test, and it skips if the platform cannot decode them. --- Lib/encodings/_iconv_codecs.py | 22 +++------------------- Lib/test/test_codecs.py | 21 ++++++++++++++------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/Lib/encodings/_iconv_codecs.py b/Lib/encodings/_iconv_codecs.py index 51c5f77e87fc46..1c701e0af423af 100644 --- a/Lib/encodings/_iconv_codecs.py +++ b/Lib/encodings/_iconv_codecs.py @@ -1,7 +1,7 @@ import codecs def create_iconv_codec(name, encoding): - from _codecs import iconv_encode, iconv_decode, iconv_state + from _codecs import iconv_encode, iconv_decode def encode(input, errors='strict'): return iconv_encode(encoding, input, errors) @@ -14,32 +14,16 @@ def encode(self, input, final=False): return iconv_encode(encoding, input, self.errors)[0] class IncrementalDecoder(codecs.BufferedIncrementalDecoder): - def __init__(self, errors='strict'): - super().__init__(errors) - self._state = iconv_state(encoding) - def _buffer_decode(self, input, errors, final): - return iconv_decode(encoding, input, errors, final, self._state) - - def reset(self): - super().reset() - self._state = iconv_state(encoding) + return iconv_decode(encoding, input, errors, final) class StreamWriter(codecs.StreamWriter): def encode(self, input, errors='strict'): return iconv_encode(encoding, input, errors) class StreamReader(codecs.StreamReader): - def __init__(self, stream, errors='strict'): - super().__init__(stream, errors) - self._state = iconv_state(encoding) - def decode(self, input, errors, final=False): - return iconv_decode(encoding, input, errors, final, self._state) - - def reset(self): - super().reset() - self._state = iconv_state(encoding) + return iconv_decode(encoding, input, errors, final) return codecs.CodecInfo( name=name, diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 24be9d1348aa22..648f7db0628508 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3824,11 +3824,20 @@ def test_stream(self): reader = codecs.getreader('iconv:' + enc)(io.BytesIO(raw)) self.assertEqual(reader.read(), text) - def test_incremental_decode_shift_state(self): + def require_stateful(self): + # Encoded here rather than by the platform: an iconv that provides the + # encoding may still be unable to encode the sample, as macOS and iOS + # cannot for ISO-2022-CN. Decoding is what these tests are about, so + # the bytes are fixed and only decoding has to work. enc = self.require(*_ICONV_STATEFUL) text = 'ABC\u4e2d\u6587DEF' - data = codecs.encode(text, 'iconv:' + enc) - self.assertEqual(codecs.decode(data, 'iconv:' + enc), text) + data = b'ABC\x1b$)A\x0eVPND\x0fDEF\x0f' + if codecs.decode(data, 'iconv:' + enc) != text: + self.skipTest('%s: this iconv cannot decode the sample' % enc) + return enc, text, data + + def test_incremental_decode_shift_state(self): + enc, text, data = self.require_stateful() dec = codecs.getincrementaldecoder('iconv:' + enc)() out = ''.join(dec.decode(data[i:i+1]) for i in range(len(data))) out += dec.decode(b'', True) @@ -3838,10 +3847,8 @@ def test_incremental_decode_shift_state(self): self.assertEqual(dec.decode(data, True), text) def test_stream_shift_state(self): - enc = self.require(*_ICONV_STATEFUL) - text = 'ABC\u4e2d\u6587DEF' - raw = codecs.encode(text, 'iconv:' + enc) - reader = codecs.getreader('iconv:' + enc)(io.BytesIO(raw)) + enc, text, data = self.require_stateful() + reader = codecs.getreader('iconv:' + enc)(io.BytesIO(data)) self.assertEqual(''.join(iter(lambda: reader.read(1), '')), text) def test_encode_kinds(self): From 17530cf785f10a7d6c8e0114409fc0e1629a3d65 Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Wed, 5 Aug 2026 22:39:44 +0300 Subject: [PATCH 3/3] Hold the conversion in a decoder type instead of a capsule The capsule was passed to iconv_decode() next to an encoding argument that it then ignored, so the two could disagree. IconvDecoder owns both, which also leaves iconv_decode() with its original signature. --- Lib/encodings/_iconv_codecs.py | 22 ++- Modules/_codecsmodule.c | 224 ++++++++++++++++++++++--------- Modules/clinic/_codecsmodule.c.h | 126 +++++++++++++---- 3 files changed, 279 insertions(+), 93 deletions(-) diff --git a/Lib/encodings/_iconv_codecs.py b/Lib/encodings/_iconv_codecs.py index 1c701e0af423af..173b53bbc2f317 100644 --- a/Lib/encodings/_iconv_codecs.py +++ b/Lib/encodings/_iconv_codecs.py @@ -1,7 +1,7 @@ import codecs def create_iconv_codec(name, encoding): - from _codecs import iconv_encode, iconv_decode + from _codecs import iconv_encode, iconv_decode, IconvDecoder def encode(input, errors='strict'): return iconv_encode(encoding, input, errors) @@ -14,16 +14,32 @@ def encode(self, input, final=False): return iconv_encode(encoding, input, self.errors)[0] class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def __init__(self, errors='strict'): + super().__init__(errors) + self._decoder = IconvDecoder(encoding) + def _buffer_decode(self, input, errors, final): - return iconv_decode(encoding, input, errors, final) + return self._decoder.decode(input, errors, final) + + def reset(self): + super().reset() + self._decoder = IconvDecoder(encoding) class StreamWriter(codecs.StreamWriter): def encode(self, input, errors='strict'): return iconv_encode(encoding, input, errors) class StreamReader(codecs.StreamReader): + def __init__(self, stream, errors='strict'): + super().__init__(stream, errors) + self._decoder = IconvDecoder(encoding) + def decode(self, input, errors, final=False): - return iconv_decode(encoding, input, errors, final) + return self._decoder.decode(input, errors, final) + + def reset(self): + super().reset() + self._decoder = IconvDecoder(encoding) return codecs.CodecInfo( name=name, diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index cffc0295ad0d3d..b71ad401654915 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -32,16 +32,45 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_codecs.h" // _PyCodec_Lookup() +#include "pycore_pymem.h" // _PyMem_Strdup() #include "pycore_unicodeobject.h" // _PyUnicode_EncodeCharmap #ifdef MS_WINDOWS #include #endif +typedef struct { + PyTypeObject *IconvDecoderType; +} _codecs_state; + +static inline _codecs_state * +get_codecs_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (_codecs_state *)state; +} + +static struct PyModuleDef codecsmodule; + +#define get_codecs_state_by_type(type) \ + (get_codecs_state(PyType_GetModuleByDef(type, &codecsmodule))) + +#ifdef HAVE_ICONV +typedef struct { + PyObject_HEAD + iconv_t cd; + char *encoding; +} iconv_decoder_object; + +#define iconv_decoder_CAST(op) ((iconv_decoder_object *)(op)) +#endif + /*[clinic input] module _codecs +class _codecs.IconvDecoder "iconv_decoder_object *" "get_codecs_state_by_type(type)->IconvDecoderType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e1390e3da3cb9deb]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=17a98f0bef095afe]*/ #include "pycore_runtime.h" #include "clinic/_codecsmodule.c.h" @@ -644,89 +673,126 @@ _codecs_code_page_decode_impl(PyObject *module, int codepage, #ifdef HAVE_ICONV -#ifdef HAVE_ICONV -#define ICONV_STATE_CAPSULE "_codecs.iconv_state" - -static void -iconv_state_destructor(PyObject *capsule) -{ - iconv_t *cdp = PyCapsule_GetPointer(capsule, ICONV_STATE_CAPSULE); - if (cdp == NULL) { - PyErr_Clear(); - return; - } - iconv_close(*cdp); - PyMem_Free(cdp); -} -#endif - /*[clinic input] -_codecs.iconv_state +@classmethod +_codecs.IconvDecoder.__new__ encoding: str / -Open an iconv conversion for decoding, to reuse across calls. +Decoder holding one iconv conversion, to reuse across calls. -The result is an opaque object. Reusing one conversion keeps the -shift state of a stateful encoding, such as ISO-2022-CN, across calls. +Reusing one conversion keeps the shift state of a stateful encoding, +such as ISO-2022-CN, from one call to the next. [clinic start generated code]*/ static PyObject * -_codecs_iconv_state_impl(PyObject *module, const char *encoding) -/*[clinic end generated code: output=4100a9b65a64d65c input=6ffbfa5bb1d2d208]*/ +_codecs_IconvDecoder_impl(PyTypeObject *type, const char *encoding) +/*[clinic end generated code: output=6e5181abedc4ae7c input=c53769050ff2b196]*/ { -#ifdef HAVE_ICONV - iconv_t *cdp = PyMem_Malloc(sizeof(iconv_t)); - if (cdp == NULL) { + char *name = _PyMem_Strdup(encoding); + if (name == NULL) { return PyErr_NoMemory(); } - *cdp = _PyUnicode_IconvOpenDecoder(encoding); - if (*cdp == (iconv_t)-1) { - PyMem_Free(cdp); + iconv_t cd = _PyUnicode_IconvOpenDecoder(encoding); + if (cd == (iconv_t)-1) { + PyMem_Free(name); return NULL; } - PyObject *capsule = PyCapsule_New(cdp, ICONV_STATE_CAPSULE, - iconv_state_destructor); - if (capsule == NULL) { - iconv_close(*cdp); - PyMem_Free(cdp); + iconv_decoder_object *self = (iconv_decoder_object *)type->tp_alloc(type, 0); + if (self == NULL) { + iconv_close(cd); + PyMem_Free(name); return NULL; } - return capsule; -#else - PyErr_SetString(PyExc_LookupError, "iconv is not available"); - return NULL; -#endif + self->cd = cd; + self->encoding = name; + return (PyObject *)self; } +/*[clinic input] +_codecs.IconvDecoder.decode + + data: Py_buffer + errors: str(accept={str, NoneType}) = None + final: bool = False + / +[clinic start generated code]*/ + +static PyObject * +_codecs_IconvDecoder_decode_impl(iconv_decoder_object *self, Py_buffer *data, + const char *errors, int final) +/*[clinic end generated code: output=0e8812b6b422fc97 input=9bea42dd438d03af]*/ +{ + Py_ssize_t consumed = data->len; + PyObject *decoded = _PyUnicode_DecodeIconv(self->encoding, data->buf, + data->len, errors, + final ? NULL : &consumed, + &self->cd); + return codec_tuple(decoded, consumed); +} + +static int +iconv_decoder_traverse(PyObject *op, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(op)); + return 0; +} + +static void +iconv_decoder_dealloc(PyObject *op) +{ + iconv_decoder_object *self = iconv_decoder_CAST(op); + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + if (self->cd != (iconv_t)-1) { + iconv_close(self->cd); + } + PyMem_Free(self->encoding); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyMethodDef iconv_decoder_methods[] = { + _CODECS_ICONVDECODER_DECODE_METHODDEF + {NULL, NULL} +}; + +static PyType_Slot iconv_decoder_slots[] = { + {Py_tp_new, _codecs_IconvDecoder}, + {Py_tp_dealloc, iconv_decoder_dealloc}, + {Py_tp_traverse, iconv_decoder_traverse}, + {Py_tp_methods, iconv_decoder_methods}, + {Py_tp_doc, (void *)_codecs_IconvDecoder__doc__}, + {0, NULL} +}; + +static PyType_Spec iconv_decoder_spec = { + .name = "_codecs.IconvDecoder", + .basicsize = sizeof(iconv_decoder_object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC), + .slots = iconv_decoder_slots, +}; + /*[clinic input] _codecs.iconv_decode encoding: str data: Py_buffer errors: str(accept={str, NoneType}) = None final: bool = False - state: object = None / [clinic start generated code]*/ static PyObject * _codecs_iconv_decode_impl(PyObject *module, const char *encoding, - Py_buffer *data, const char *errors, int final, - PyObject *state) -/*[clinic end generated code: output=ed99087a9b21d007 input=b23f4298c963f9c5]*/ + Py_buffer *data, const char *errors, int final) +/*[clinic end generated code: output=6c6145a9decc2ba8 input=d15a04d7d3a3e0cd]*/ { - iconv_t *cdp = NULL; - if (state != Py_None) { - cdp = PyCapsule_GetPointer(state, ICONV_STATE_CAPSULE); - if (cdp == NULL) { - return NULL; - } - } Py_ssize_t consumed = data->len; PyObject *decoded = _PyUnicode_DecodeIconv(encoding, data->buf, data->len, errors, - final ? NULL : &consumed, cdp); + final ? NULL : &consumed, NULL); return codec_tuple(decoded, consumed); } @@ -1220,7 +1286,6 @@ static PyMethodDef _codecs_functions[] = { _CODECS_CODE_PAGE_DECODE_METHODDEF _CODECS_ICONV_ENCODE_METHODDEF _CODECS_ICONV_DECODE_METHODDEF - _CODECS_ICONV_STATE_METHODDEF _CODECS_REGISTER_ERROR_METHODDEF _CODECS__UNREGISTER_ERROR_METHODDEF _CODECS_LOOKUP_ERROR_METHODDEF @@ -1228,23 +1293,60 @@ static PyMethodDef _codecs_functions[] = { {NULL, NULL} /* sentinel */ }; +static int +_codecs_exec(PyObject *module) +{ +#ifdef HAVE_ICONV + _codecs_state *state = get_codecs_state(module); + state->IconvDecoderType = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &iconv_decoder_spec, NULL); + if (state->IconvDecoderType == NULL) { + return -1; + } + if (PyModule_AddType(module, state->IconvDecoderType) < 0) { + return -1; + } +#endif + return 0; +} + +static int +_codecs_traverse(PyObject *module, visitproc visit, void *arg) +{ + Py_VISIT(get_codecs_state(module)->IconvDecoderType); + return 0; +} + +static int +_codecs_clear(PyObject *module) +{ + Py_CLEAR(get_codecs_state(module)->IconvDecoderType); + return 0; +} + +static void +_codecs_free(void *module) +{ + (void)_codecs_clear((PyObject *)module); +} + static PyModuleDef_Slot _codecs_slots[] = { _Py_ABI_SLOT, + {Py_mod_exec, _codecs_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; static struct PyModuleDef codecsmodule = { - PyModuleDef_HEAD_INIT, - "_codecs", - NULL, - 0, - _codecs_functions, - _codecs_slots, - NULL, - NULL, - NULL + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_codecs", + .m_size = sizeof(_codecs_state), + .m_methods = _codecs_functions, + .m_slots = _codecs_slots, + .m_traverse = _codecs_traverse, + .m_clear = _codecs_clear, + .m_free = _codecs_free, }; PyMODINIT_FUNC diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 4507f61ab2c287..000f346c9d3126 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -1631,33 +1631,38 @@ _codecs_code_page_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar #if defined(HAVE_ICONV) -PyDoc_STRVAR(_codecs_iconv_state__doc__, -"iconv_state($module, encoding, /)\n" +PyDoc_STRVAR(_codecs_IconvDecoder__doc__, +"IconvDecoder(encoding, /)\n" "--\n" "\n" -"Open an iconv conversion for decoding, to reuse across calls.\n" +"Decoder holding one iconv conversion, to reuse across calls.\n" "\n" -"The result is an opaque object. Reusing one conversion keeps the\n" -"shift state of a stateful encoding, such as ISO-2022-CN, across calls."); - -#define _CODECS_ICONV_STATE_METHODDEF \ - {"iconv_state", (PyCFunction)_codecs_iconv_state, METH_O, _codecs_iconv_state__doc__}, +"Reusing one conversion keeps the shift state of a stateful encoding,\n" +"such as ISO-2022-CN, from one call to the next."); static PyObject * -_codecs_iconv_state_impl(PyObject *module, const char *encoding); +_codecs_IconvDecoder_impl(PyTypeObject *type, const char *encoding); static PyObject * -_codecs_iconv_state(PyObject *module, PyObject *arg) +_codecs_IconvDecoder(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; + PyTypeObject *base_tp = get_codecs_state_by_type(type)->IconvDecoderType; const char *encoding; - if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("iconv_state", "argument", "str", arg); + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("IconvDecoder", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("IconvDecoder", PyTuple_GET_SIZE(args), 1, 1)) { + goto exit; + } + if (!PyUnicode_Check(PyTuple_GET_ITEM(args, 0))) { + _PyArg_BadArgument("IconvDecoder", "argument 1", "str", PyTuple_GET_ITEM(args, 0)); goto exit; } Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(arg, &encoding_length); + encoding = PyUnicode_AsUTF8AndSize(PyTuple_GET_ITEM(args, 0), &encoding_length); if (encoding == NULL) { goto exit; } @@ -1665,7 +1670,7 @@ _codecs_iconv_state(PyObject *module, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _codecs_iconv_state_impl(module, encoding); + return_value = _codecs_IconvDecoder_impl(type, encoding); exit: return return_value; @@ -1675,9 +1680,78 @@ _codecs_iconv_state(PyObject *module, PyObject *arg) #if defined(HAVE_ICONV) +PyDoc_STRVAR(_codecs_IconvDecoder_decode__doc__, +"decode($self, data, errors=None, final=False, /)\n" +"--\n" +"\n"); + +#define _CODECS_ICONVDECODER_DECODE_METHODDEF \ + {"decode", _PyCFunction_CAST(_codecs_IconvDecoder_decode), METH_FASTCALL, _codecs_IconvDecoder_decode__doc__}, + +static PyObject * +_codecs_IconvDecoder_decode_impl(iconv_decoder_object *self, Py_buffer *data, + const char *errors, int final); + +static PyObject * +_codecs_IconvDecoder_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + const char *errors = NULL; + int final = 0; + + if (!_PyArg_CheckPositional("decode", nargs, 1, 3)) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (args[1] == Py_None) { + errors = NULL; + } + else if (PyUnicode_Check(args[1])) { + Py_ssize_t errors_length; + errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + if (errors == NULL) { + goto exit; + } + if (strlen(errors) != (size_t)errors_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + } + else { + _PyArg_BadArgument("decode", "argument 2", "str or None", args[1]); + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + final = PyObject_IsTrue(args[2]); + if (final < 0) { + goto exit; + } +skip_optional: + return_value = _codecs_IconvDecoder_decode_impl((iconv_decoder_object *)self, &data, errors, final); + +exit: + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); + } + + return return_value; +} + +#endif /* defined(HAVE_ICONV) */ + +#if defined(HAVE_ICONV) + PyDoc_STRVAR(_codecs_iconv_decode__doc__, -"iconv_decode($module, encoding, data, errors=None, final=False,\n" -" state=None, /)\n" +"iconv_decode($module, encoding, data, errors=None, final=False, /)\n" "--\n" "\n"); @@ -1686,8 +1760,7 @@ PyDoc_STRVAR(_codecs_iconv_decode__doc__, static PyObject * _codecs_iconv_decode_impl(PyObject *module, const char *encoding, - Py_buffer *data, const char *errors, int final, - PyObject *state); + Py_buffer *data, const char *errors, int final); static PyObject * _codecs_iconv_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -1697,9 +1770,8 @@ _codecs_iconv_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_buffer data = {NULL, NULL}; const char *errors = NULL; int final = 0; - PyObject *state = Py_None; - if (!_PyArg_CheckPositional("iconv_decode", nargs, 2, 5)) { + if (!_PyArg_CheckPositional("iconv_decode", nargs, 2, 4)) { goto exit; } if (!PyUnicode_Check(args[0])) { @@ -1746,12 +1818,8 @@ _codecs_iconv_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (final < 0) { goto exit; } - if (nargs < 5) { - goto skip_optional; - } - state = args[4]; skip_optional: - return_value = _codecs_iconv_decode_impl(module, encoding, &data, errors, final, state); + return_value = _codecs_iconv_decode_impl(module, encoding, &data, errors, final); exit: /* Cleanup for data */ @@ -3065,9 +3133,9 @@ _codecs__normalize_encoding(PyObject *module, PyObject *const *args, Py_ssize_t #define _CODECS_CODE_PAGE_DECODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_DECODE_METHODDEF) */ -#ifndef _CODECS_ICONV_STATE_METHODDEF - #define _CODECS_ICONV_STATE_METHODDEF -#endif /* !defined(_CODECS_ICONV_STATE_METHODDEF) */ +#ifndef _CODECS_ICONVDECODER_DECODE_METHODDEF + #define _CODECS_ICONVDECODER_DECODE_METHODDEF +#endif /* !defined(_CODECS_ICONVDECODER_DECODE_METHODDEF) */ #ifndef _CODECS_ICONV_DECODE_METHODDEF #define _CODECS_ICONV_DECODE_METHODDEF @@ -3088,4 +3156,4 @@ _codecs__normalize_encoding(PyObject *module, PyObject *const *args, Py_ssize_t #ifndef _CODECS_ICONV_ENCODE_METHODDEF #define _CODECS_ICONV_ENCODE_METHODDEF #endif /* !defined(_CODECS_ICONV_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=30b2a2c3eb23dfc1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8b51735946040ff2 input=a9049054013a1b77]*/