feat: add max_retries to the messages handler factories

packages/claude-messages/api-report/launchdarkly_ai_claude_messages.txt

@@ -1,3 +1,3 @@
 # Public API of launchdarkly_ai_claude_messages. Generated by `make api-report`; do not edit.
 claude_messages(config_key: str, user_input: str, context: LDContext, **kwargs: Any) -> Any
-create_claude_messages_handler(*, capture_content: bool | None = None) -> ProviderHandler
+create_claude_messages_handler(*, capture_content: bool | None = None, max_retries: int | None = None) -> ProviderHandler

packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py

@@ -25,6 +25,7 @@
     set_tool_call_content_attributes,
     to_semconv_finish_reason,
     tool_result_string,
+    validate_max_retries,
 )
 from launchdarkly_ai_server.agui import TurnEmitter
 from launchdarkly_ai_server.lifecycle import effective_capture_content
@@ -529,7 +530,7 @@ def _is_coroutine(fn: Any) -> bool:
 
 
 def create_claude_messages_handler(
-    *, capture_content: bool | None = None
+    *, capture_content: bool | None = None, max_retries: int | None = None
 ) -> ProviderHandler:
     """Creates a ``ProviderHandler`` for Anthropic Claude (messages API).
 
@@ -541,22 +542,37 @@ def create_claude_messages_handler(
     for this handler. Conversation content is PII, so a run emits only metadata, meaning
     models, token counts, timings and tool names, until capture is turned on. Turning this
     on sends the text of every request and response to whatever collector the SDK is pointed at.
+
+    Set *max_retries* to change how many times the ``AsyncAnthropic`` client retries a failed
+    request (a rate limit, an overload, a timeout, a 5xx) before the error reaches the handler.
+    ``None`` (the default) leaves the client's own default of 2 in place. Raise it when many calls
+    share one rate-limit window, as an evaluation run with many concurrent workers does: two
+    retries is thin when a whole batch hits the same window at once. The retries happen inside the
+    client, so a retried turn is still one ``chat`` span. Raises ``TypeError`` or ``ValueError``
+    when it is not a non-negative int.
     """
     report_usage(
         "claude-messages.createClaudeMessagesHandler", PACKAGE_NAME, __version__
     )
-    return _create_claude_messages_handler(capture_content=capture_content)
+    return _create_claude_messages_handler(
+        capture_content=capture_content, max_retries=max_retries
+    )
 
 
 def _create_claude_messages_handler(
-    *, capture_content: bool | None = None
+    *, capture_content: bool | None = None, max_retries: int | None = None
 ) -> ProviderHandler:
     """Non-reporting :func:`create_claude_messages_handler`, used by this package's wrappers."""
+    max_retries = validate_max_retries(max_retries)
     content_override = capture_content
     import importlib
 
     anthropic_mod = importlib.import_module("anthropic")
-    client = anthropic_mod.AsyncAnthropic()
+    # Passed only when set, so an omitted value keeps the vendor default rather than a copy of it.
+    client_options: dict[str, Any] = {}
+    if max_retries is not None:
+        client_options["max_retries"] = max_retries
+    client = anthropic_mod.AsyncAnthropic(**client_options)
 
     async def _call_impl(
         config: AiConfigRep,
@@ -950,14 +966,17 @@ def claude_messages(
     **kwargs: Any,
 ) -> Any:
     """Convenience wrapper: creates a handler and calls config(...).invoke()."""
-    # Both are lifted out of kwargs: capture_content configures the handler, variables belong to
-    # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller
-    # asking for content on spans got a TypeError instead of content.
+    # All three are lifted out of kwargs: capture_content and max_retries configure the handler,
+    # variables belong to the invocation. Leaving any in would pass it to config(), which takes
+    # none of them, so a caller asking for content on spans got a TypeError instead of content.
     report_usage("claude-messages.claudeMessages", PACKAGE_NAME, __version__)
     variables = kwargs.pop("variables", None)
     capture_content = kwargs.pop("capture_content", None)
+    max_retries = kwargs.pop("max_retries", None)
     return _config(
         key=config_key,
-        handler=_create_claude_messages_handler(capture_content=capture_content),
+        handler=_create_claude_messages_handler(
+            capture_content=capture_content, max_retries=max_retries
+        ),
         **kwargs,
     ).invoke(user_input, context, variables=variables)

packages/claude-messages/tests/test_handler.py

@@ -3127,3 +3127,124 @@ async def test_effort_still_lands_in_output_config_when_output_config_is_malform
         await create_claude_messages_handler()(config, "q", {}, {})
         kwargs = mock_anthropic.messages.create.call_args.kwargs
         assert kwargs["output_config"] == {"effort": "low"}
+
+
+class TestMaxRetries:
+    """`max_retries` reaches the `AsyncAnthropic` client the factory builds, and only when set.
+
+    The client's own retries are the right layer for rate limits, so an omitted value must leave the
+    vendor default in place rather than replace it, and nothing may ever turn retries off.
+    """
+
+    def _build(
+        self, monkeypatch: pytest.MonkeyPatch, **kwargs: Any
+    ) -> tuple[Any, Any, MagicMock]:
+        import anthropic
+
+        from launchdarkly_ai_claude_messages import create_claude_messages_handler
+
+        monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key")
+        real_client = anthropic.AsyncAnthropic
+        built: list[Any] = []
+
+        def _construct(**options: Any) -> Any:
+            built.append(real_client(**options))
+            return built[-1]
+
+        with patch("anthropic.AsyncAnthropic", side_effect=_construct) as ctor:
+            handler = create_claude_messages_handler(**kwargs)
+        return handler, built[0], ctor
+
+    def test_value_reaches_the_constructed_client(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        _handler, client, _ctor = self._build(monkeypatch, max_retries=7)
+        assert client.max_retries == 7
+
+    def test_omitted_value_keeps_the_vendor_default(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        import anthropic
+
+        _handler, client, ctor = self._build(monkeypatch)
+        assert "max_retries" not in ctor.call_args.kwargs
+        assert client.max_retries == anthropic.DEFAULT_MAX_RETRIES
+
+    def test_explicit_zero_is_the_callers_choice(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        _handler, client, _ctor = self._build(monkeypatch, max_retries=0)
+        assert client.max_retries == 0
+
+    @pytest.mark.parametrize("value", [True, 2.5, "3"])
+    def test_non_integer_is_refused_at_construction(self, value: Any) -> None:
+        from launchdarkly_ai_claude_messages import create_claude_messages_handler
+
+        with patch("anthropic.AsyncAnthropic") as ctor:
+            with pytest.raises(TypeError, match="max_retries"):
+                create_claude_messages_handler(max_retries=value)
+        ctor.assert_not_called()
+
+    def test_negative_is_refused_at_construction(self) -> None:
+        from launchdarkly_ai_claude_messages import create_claude_messages_handler
+
+        with patch("anthropic.AsyncAnthropic") as ctor:
+            with pytest.raises(ValueError, match="max_retries"):
+                create_claude_messages_handler(max_retries=-1)
+        ctor.assert_not_called()
+
+    async def test_config_cannot_set_it(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        import launchdarkly_ai_claude_messages.spans as spans_mod
+
+        handler, client, _ctor = self._build(monkeypatch, max_retries=3)
+        config = {
+            **CONFIG,
+            "model": {**CONFIG["model"], "parameters": {"max_retries": 50}},
+        }
+        create = AsyncMock(return_value=_anthropic_response([_text_block("hi")]))
+        with (
+            patch.object(spans_mod, "_HAS_OTEL", False),
+            patch.object(client.messages, "create", create),
+        ):
+            await handler(config, "q", {}, {})
+        assert client.max_retries == 3
+        # A per-request max_retries would override the client's, so it must not be forwarded.
+        assert "max_retries" not in create.call_args.kwargs
+
+    async def test_invoke_and_stream_share_the_configured_client(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        import launchdarkly_ai_claude_messages.spans as spans_mod
+
+        handler, client, ctor = self._build(monkeypatch, max_retries=7)
+        create = AsyncMock(return_value=_anthropic_response([_text_block("hi")]))
+        stream = MagicMock(return_value=_make_stream_context(["hi"])[0])
+        with (
+            patch.object(spans_mod, "_HAS_OTEL", False),
+            patch.object(client.messages, "create", create),
+            patch.object(client.messages, "stream", stream),
+        ):
+            await handler(CONFIG, "q", {}, {})
+            async for _event in await handler.stream(CONFIG, "q"):
+                pass
+        create.assert_awaited_once()
+        assert "max_retries" not in create.call_args.kwargs
+        stream.assert_called_once()
+        assert "max_retries" not in stream.call_args.kwargs
+        # One client, built once, serves both paths, so both get its retry count.
+        assert ctor.call_count == 1
+        assert client.max_retries == 7
+
+    def test_convenience_wrapper_forwards_it_to_the_factory(self) -> None:
+        import launchdarkly_ai_claude_messages.handler as handler_mod
+
+        factory = MagicMock()
+        fake_config = MagicMock()
+        with (
+            patch.object(handler_mod, "_create_claude_messages_handler", factory),
+            patch.object(handler_mod, "_config", fake_config),
+        ):
+            handler_mod.claude_messages("k", "q", {}, max_retries=6)
+        assert factory.call_args.kwargs["max_retries"] == 6
+        # config() takes no such argument, so leaving it in kwargs would raise TypeError.
+        assert "max_retries" not in fake_config.call_args.kwargs

packages/client/api-report/launchdarkly_ai_server.txt

@@ -326,6 +326,7 @@ UsageDict  [class, @dataclass]
   input_details: InputTokenDetails | None = None
   output: int = 0
   total: int = 0
+validate_max_retries(max_retries: int | None) -> int | None
 VariationMeta = dict[str, Any]
 watch_skills(skills: Sequence[Skill | SkillReference | str] | str | SkillSource, root: str | os.PathLike[str], *, prune: bool = True, timeout: float = 10.0, on_unavailable: OnUnavailable = 'keep', debounce: float = DEFAULT_DEBOUNCE_SECONDS, on_reconcile: Callable[[ReconcileReport], Any] | None = None) -> tuple[ReconcileReport, SkillWatcher]  [async]
 wrap_tool_handlers(tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None, user_context: LDContext, track_data: TrackData) -> dict[str, Callable[..., Any]]

packages/client/src/launchdarkly_ai_server/__init__.py

@@ -146,6 +146,7 @@
     set_usage_span_attributes,
     to_ld_context,
     to_usage_dict,
+    validate_max_retries,
 )
 from .utils import _create_handler as _create_handler
 
@@ -221,6 +222,7 @@
     "parse_usage",
     "set_ld_span_attributes",
     "to_ld_context",
+    "validate_max_retries",
     # history
     "compose_history",
     "content_to_text",

packages/client/src/launchdarkly_ai_server/utils.py

@@ -72,6 +72,28 @@ def model_parameters(config: AiConfigRep) -> dict[str, Any]:
     return dict(parameters) if isinstance(parameters, dict) else {}
 
 
+def validate_max_retries(max_retries: int | None) -> int | None:
+    """Checks a handler factory's ``max_retries`` option and returns it unchanged.
+
+    ``None`` means the option was not given, so the vendor default applies. Any other value must be
+    a non-negative ``int``; ``0`` is a valid count. ``bool`` is refused even though it subclasses
+    ``int``, since ``max_retries=True`` is a mistake rather than one retry. Factories call this at
+    construction, so a bad value fails when the handler is built rather than on its first call.
+
+    Raises ``TypeError`` for a non-integer and ``ValueError`` for a negative integer. Both messages
+    name the option.
+    """
+    if max_retries is None:
+        return None
+    if isinstance(max_retries, bool) or not isinstance(max_retries, int):
+        raise TypeError(
+            f"max_retries must be a non-negative int or None, got {type(max_retries).__name__}"
+        )
+    if max_retries < 0:
+        raise ValueError(f"max_retries must be non-negative, got {max_retries}")
+    return max_retries
+
+
 def collapse_messages_to_instructions(config: AiConfigRep) -> AiConfigRep:
     """
     When only an agent handler is available for a messages-mode config, collapse

packages/client/tests/test_public_surface.py

@@ -115,6 +115,7 @@
         "to_semconv_finish_reason",
         "to_usage_dict",
         "tool_result_string",
+        "validate_max_retries",
         "wrap_tool_handlers",
         # Agent Skills
         "set_skill_store",

packages/client/tests/test_utils.py

@@ -17,6 +17,7 @@
     parse_json_with_possible_fences,
     parse_template,
     parse_usage,
+    validate_max_retries,
 )
 from launchdarkly_ai_server.utils import make_graph_track_data, model_parameters
 
@@ -435,3 +436,21 @@ def test_never_reads_custom(self) -> None:
         result = model_parameters(config)
         assert "secret" not in result
         assert "custom" not in result
+
+
+class TestValidateMaxRetries:
+    """TESTING.md A.22: a factory refuses a value that is not a non-negative int, naming the option."""
+
+    @pytest.mark.parametrize("value", [None, 0, 1, 7, 50])
+    def test_accepts_none_and_non_negative_ints(self, value: int | None) -> None:
+        assert validate_max_retries(value) == value
+
+    @pytest.mark.parametrize("value", [True, False, 2.0, "3", [3]])
+    def test_refuses_non_integers_with_type_error(self, value: Any) -> None:
+        with pytest.raises(TypeError, match="max_retries"):
+            validate_max_retries(value)
+
+    @pytest.mark.parametrize("value", [-1, -50])
+    def test_refuses_negatives_with_value_error(self, value: int) -> None:
+        with pytest.raises(ValueError, match="max_retries"):
+            validate_max_retries(value)

packages/langchain-messages/api-report/launchdarkly_ai_langchain_messages.txt

@@ -1,3 +1,3 @@
 # Public API of launchdarkly_ai_langchain_messages. Generated by `make api-report`; do not edit.
-create_langchain_messages_handler(llm: Any = None, *, capture_content: bool | None = None) -> ProviderHandler
+create_langchain_messages_handler(llm: Any = None, *, capture_content: bool | None = None, max_retries: int | None = None) -> ProviderHandler
 langchain_messages(config_key: str, user_input: str, context: LDContext, llm: Any = None, **kwargs: Any) -> Any