From 3d214132e2f52b7de5eaf1acbb73a4c80811cd03 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 4 Aug 2026 23:13:32 +0300 Subject: [PATCH 1/2] gh-63299: Escape unencodable characters in pprint() pprint.pp(), pprint.pprint() and PrettyPrinter.pprint() failed with UnicodeEncodeError if the output could not be encoded in the encoding of the output stream. Such characters are now escaped with backslashes, as sys.displayhook does. The stream is only wrapped if it encodes the written text and uses the strict error handler. Co-Authored-By: Claude Opus 5 (1M context) --- Doc/library/pprint.rst | 9 ++++ Lib/pprint.py | 36 ++++++++++++++- Lib/test/test_pprint.py | 44 +++++++++++++++++++ ...6-08-04-22-40-00.gh-issue-63299.pprBsl.rst | 5 +++ 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-04-22-40-00.gh-issue-63299.pprBsl.rst diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 4f043fbb3a46dff..99c7a66dfaf982b 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -46,6 +46,8 @@ Functions A file-like object to which the output will be written by calling its :meth:`!write` method. If ``None`` (the default), :data:`sys.stdout` is used. + Characters which cannot be encoded in the encoding of the stream are + escaped with backslashes, unless the stream itself handles them. :type stream: :term:`file-like object` | None :param int indent: @@ -101,6 +103,10 @@ Functions .. versionadded:: 3.8 + .. versionchanged:: next + Unencodable characters are escaped instead of raising + :exc:`UnicodeEncodeError`. + .. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ compact=False, expand=False, sort_dicts=True, \ @@ -238,6 +244,9 @@ PrettyPrinter Objects Print the formatted representation of *object* on the configured stream, followed by a newline. + Characters which cannot be encoded in the encoding of the stream are + escaped with backslashes, unless the stream itself handles them. + The following methods provide the implementations for the corresponding functions of the same names. Using these methods on an instance is slightly more efficient since new :class:`PrettyPrinter` objects don't need to be diff --git a/Lib/pprint.py b/Lib/pprint.py index 7355021998081dc..5b74b221c435789 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -111,6 +111,37 @@ def _safe_tuple(t): return _safe_key(t[0]), _safe_key(t[1]) +class _EscapingWriter: + """Wrapper which escapes characters unencodable in the stream encoding.""" + + def __init__(self, stream, encoding): + self._stream = stream + self._encoding = encoding + + def write(self, text): + text = text.encode(self._encoding, 'backslashreplace') + return self._stream.write(text.decode(self._encoding)) + + def __getattr__(self, name): + return getattr(self._stream, name) + + +def _escape_unencodable(stream): + """Return a stream which never fails on unencodable characters. + + The output is intended to be read by humans, so it is better to escape + unencodable characters than to fail. Streams which do not encode the + written text, or which already handle unencodable characters, are + returned unchanged. + """ + encoding = getattr(stream, 'encoding', None) + if encoding is None: + return stream + if getattr(stream, 'errors', 'strict') != 'strict': + return stream + return _EscapingWriter(stream, encoding) + + class PrettyPrinter: def __init__(self, indent=1, width=80, depth=None, stream=None, *, compact=False, expand=False, sort_dicts=True, @@ -171,8 +202,9 @@ def __init__(self, indent=1, width=80, depth=None, stream=None, *, def pprint(self, object): if self._stream is not None: - self._format(object, self._stream, 0, 0, {}, 0) - self._stream.write("\n") + stream = _escape_unencodable(self._stream) + self._format(object, stream, 0, 0, {}, 0) + stream.write("\n") def pformat(self, object): sio = _StringIO() diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 041c2072b9e253a..70c18bba6478c55 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -1147,6 +1147,50 @@ def test_str_wrap(self): formatted = pprint.pformat([special] * 2, width=width) self.assertEqual(eval(formatted), [special] * 2) + def test_unencodable(self): + # with encoding and buffer + with io.BytesIO() as bio, \ + io.TextIOWrapper(bio, encoding='latin1') as stream: + stream.write('\xab') + pprint.pprint('\xa3\u20ac', stream) + stream.flush() + self.assertEqual(bio.getvalue(), b"\xab'\xa3\\u20ac'\n") + stream.write('\xbb') + stream.flush() + self.assertEqual(bio.getvalue(), b"\xab'\xa3\\u20ac'\n\xbb") + # with encoding but without buffer + class MockWriter(list): + encoding = 'latin1' + errors = 'strict' + write = list.append + stream = MockWriter() + stream.write('\xab') + pprint.pprint('\xa3\u20ac', stream) + self.assertEqual(''.join(stream), "\xab'\xa3\\u20ac'\n") + # without encoding + with io.StringIO() as stream: + stream.write('\xab') + pprint.pprint('\xa3\u20ac', stream) + self.assertEqual(stream.getvalue(), "\xab'\xa3\u20ac'\n") + # the error handler of the stream is used if it is not strict + with io.BytesIO() as bio, \ + io.TextIOWrapper(bio, encoding='latin1', + errors='replace') as stream: + pprint.pprint('\u20ac', stream) + stream.flush() + self.assertEqual(bio.getvalue(), b"'?'\n") + + def test_unencodable_repr(self): + class Surrogate: + def __repr__(self): + return '\udcff' + + with io.BytesIO() as bio, \ + io.TextIOWrapper(bio, encoding='utf-8') as stream: + pprint.pprint(Surrogate(), stream) + stream.flush() + self.assertEqual(bio.getvalue(), b"\\udcff\n") + def test_compact(self): o = ([list(range(i * i)) for i in range(5)] + [list(range(i)) for i in range(6)]) diff --git a/Misc/NEWS.d/next/Library/2026-08-04-22-40-00.gh-issue-63299.pprBsl.rst b/Misc/NEWS.d/next/Library/2026-08-04-22-40-00.gh-issue-63299.pprBsl.rst new file mode 100644 index 000000000000000..ba82962148256f1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-04-22-40-00.gh-issue-63299.pprBsl.rst @@ -0,0 +1,5 @@ +:func:`pprint.pp`, :func:`pprint.pprint` and +:meth:`pprint.PrettyPrinter.pprint` no longer fail with +:exc:`UnicodeEncodeError` if the output contains characters unencodable in the +encoding of the output stream. Such characters are now escaped with +backslashes. From 61c09468f435f66d460e2aedc4353b5d3a635a3e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Aug 2026 12:10:29 +0300 Subject: [PATCH 2/2] Fix the tests on Windows TextIOWrapper translates "\n" to os.linesep if newline is None. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/test_pprint.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 70c18bba6478c55..f73b3fa0ebe78de 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -1150,7 +1150,8 @@ def test_str_wrap(self): def test_unencodable(self): # with encoding and buffer with io.BytesIO() as bio, \ - io.TextIOWrapper(bio, encoding='latin1') as stream: + io.TextIOWrapper(bio, encoding='latin1', + newline='') as stream: stream.write('\xab') pprint.pprint('\xa3\u20ac', stream) stream.flush() @@ -1174,8 +1175,8 @@ class MockWriter(list): self.assertEqual(stream.getvalue(), "\xab'\xa3\u20ac'\n") # the error handler of the stream is used if it is not strict with io.BytesIO() as bio, \ - io.TextIOWrapper(bio, encoding='latin1', - errors='replace') as stream: + io.TextIOWrapper(bio, encoding='latin1', errors='replace', + newline='') as stream: pprint.pprint('\u20ac', stream) stream.flush() self.assertEqual(bio.getvalue(), b"'?'\n") @@ -1186,7 +1187,8 @@ def __repr__(self): return '\udcff' with io.BytesIO() as bio, \ - io.TextIOWrapper(bio, encoding='utf-8') as stream: + io.TextIOWrapper(bio, encoding='utf-8', + newline='') as stream: pprint.pprint(Surrogate(), stream) stream.flush() self.assertEqual(bio.getvalue(), b"\\udcff\n")