From c412417646698f06d63a5643adeb8a48271d7455 Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Thu, 13 Aug 2026 15:20:00 -0700 Subject: [PATCH 1/3] fix(evals): never emit blank mocker error messages (SRE-640375) Timeout/cancellation exceptions (asyncio.TimeoutError, CancelledError, httpx timeouts) stringify to an empty string, so the input mocker's 'Failed to generate input: ' wrap and the LLM mocker's str(e) wrap produced blank, undiagnosable ErrorMessages in EvalRun.Failed telemetry. Format wrapped exceptions as 'TypeName: message', falling back to the type name alone when the message is empty. Co-Authored-By: Claude Fable 5 --- .../src/uipath/eval/mocks/_input_mocker.py | 6 ++- .../src/uipath/eval/mocks/_llm_mocker.py | 5 +- .../uipath/src/uipath/eval/mocks/_mocker.py | 11 ++++ .../eval/mocks/test_mock_error_messages.py | 51 +++++++++++++++++++ 4 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 packages/uipath/tests/cli/eval/mocks/test_mock_error_messages.py diff --git a/packages/uipath/src/uipath/eval/mocks/_input_mocker.py b/packages/uipath/src/uipath/eval/mocks/_input_mocker.py index a542fc7ad..2256938f1 100644 --- a/packages/uipath/src/uipath/eval/mocks/_input_mocker.py +++ b/packages/uipath/src/uipath/eval/mocks/_input_mocker.py @@ -14,7 +14,7 @@ from .._execution_context import eval_set_run_id_context from ._mock_context import cache_manager_context -from ._mocker import UiPathInputMockingError +from ._mocker import UiPathInputMockingError, format_exception_message from ._structured_output import generate_structured_output from ._types import ( InputMockingStrategy, @@ -155,4 +155,6 @@ async def generate_llm_input( except UiPathInputMockingError: raise except Exception as e: - raise UiPathInputMockingError(f"Failed to generate input: {str(e)}") from e + raise UiPathInputMockingError( + f"Failed to generate input: {format_exception_message(e)}" + ) from e diff --git a/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py b/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py index 5c9a0cf38..559a2e32f 100644 --- a/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py +++ b/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py @@ -28,6 +28,7 @@ T, UiPathMockResponseGenerationError, UiPathNoMockFoundError, + format_exception_message, ) from ._structured_output import generate_structured_output from ._types import ( @@ -230,7 +231,9 @@ async def response( return result except Exception as e: - raise UiPathMockResponseGenerationError(str(e)) from e + raise UiPathMockResponseGenerationError( + format_exception_message(e) + ) from e else: raise UiPathNoMockFoundError(f"Method '{function_name}' is not simulated.") diff --git a/packages/uipath/src/uipath/eval/mocks/_mocker.py b/packages/uipath/src/uipath/eval/mocks/_mocker.py index 99e5da1b2..45e0da628 100644 --- a/packages/uipath/src/uipath/eval/mocks/_mocker.py +++ b/packages/uipath/src/uipath/eval/mocks/_mocker.py @@ -22,6 +22,17 @@ async def response( raise NotImplementedError() +def format_exception_message(e: BaseException) -> str: + """Format an exception for wrapped error messages, always naming its type. + + Timeout and cancellation exceptions (``asyncio.TimeoutError``, + ``CancelledError``, httpx timeouts) often have an empty ``str``, which + otherwise produces blank, undiagnosable error messages downstream. + """ + message = str(e).strip() + return f"{type(e).__name__}: {message}" if message else type(e).__name__ + + class UiPathNoMockFoundError(Exception): """Exception when a mocker is unable to find a match with the invocation. This is a signal to invoke the real function.""" diff --git a/packages/uipath/tests/cli/eval/mocks/test_mock_error_messages.py b/packages/uipath/tests/cli/eval/mocks/test_mock_error_messages.py new file mode 100644 index 000000000..23bc0042e --- /dev/null +++ b/packages/uipath/tests/cli/eval/mocks/test_mock_error_messages.py @@ -0,0 +1,51 @@ +import asyncio +from typing import Any + +import pytest +from _pytest.monkeypatch import MonkeyPatch + +from uipath.eval.mocks import _input_mocker +from uipath.eval.mocks._input_mocker import generate_llm_input +from uipath.eval.mocks._mocker import ( + UiPathInputMockingError, + format_exception_message, +) +from uipath.eval.mocks._types import InputMockingStrategy + + +def test_format_exception_message_with_message(): + assert format_exception_message(ValueError("bad value")) == "ValueError: bad value" + + +def test_format_exception_message_empty_str(): + # asyncio.TimeoutError and CancelledError stringify to "" — the type name + # must still be surfaced so the wrapped message is never blank. + assert format_exception_message(asyncio.TimeoutError()) == "TimeoutError" + assert format_exception_message(asyncio.CancelledError()) == "CancelledError" + + +def test_format_exception_message_whitespace_only(): + assert format_exception_message(RuntimeError(" ")) == "RuntimeError" + + +@pytest.mark.asyncio +async def test_generate_llm_input_wraps_empty_str_exception( + monkeypatch: MonkeyPatch, +): + monkeypatch.setenv("UIPATH_URL", "https://example.com") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "test-token") + + async def raise_timeout(*args: Any, **kwargs: Any) -> Any: + raise asyncio.TimeoutError() + + monkeypatch.setattr(_input_mocker, "generate_structured_output", raise_timeout) + + with pytest.raises(UiPathInputMockingError) as exc_info: + await generate_llm_input( + mocking_strategy=InputMockingStrategy(prompt="generate something"), + input_schema={"type": "object", "properties": {}}, + expected_behavior="", + expected_output={}, + ) + + assert str(exc_info.value) == "Failed to generate input: TimeoutError" From 6dc2aacfecfe5763512b6d4b7843f7a45551c06b Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Thu, 13 Aug 2026 15:39:14 -0700 Subject: [PATCH 2/3] chore: publish uipath 2.14.5 Co-Authored-By: Claude Fable 5 --- packages/uipath/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 65f408502..2ac4f3d35 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.14.4" +version = "2.14.5" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" From 903e0feb5067c3202826f9ff841a899a3224eb10 Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Thu, 13 Aug 2026 15:41:49 -0700 Subject: [PATCH 3/3] chore: update uv.lock for 2.14.5 Co-Authored-By: Claude Fable 5 --- packages/uipath/uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index eee282f4b..106ad76c8 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2599,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.14.4" +version = "2.14.5" source = { editable = "." } dependencies = [ { name = "applicationinsights" },