From 3c011bcbf4fbd0d76b035140af92aceaf8782316 Mon Sep 17 00:00:00 2001 From: Caleb Martin Date: Tue, 8 Sep 2026 15:23:59 -0700 Subject: [PATCH] feat(context-grounding): let agents opt in to searching an ingesting index Searching a context index while it was ingesting raised IngestionInProgressException unconditionally, so an agent pointed at an index that was still ingesting failed outright instead of retrieving the documents already indexed. ECS serves partial results during ingestion, but there was no way to ask for them. Add an opt-in flag rather than dropping the guard, so the default stays the current fail-fast behaviour and the choice can surface as a toggle in the agent designer: - unified_search and unified_search_async take search_during_ingestion (default False) and only raise when it is False. - AgentContextSettings gains searchDuringIngestion (default False), the agent.json field the UI writes. - The CLI search command exposes it as --search-during-ingestion, keeping the SDK/CLI alignment contract satisfied. Sync unified_search never had the guard its async twin did, which looks like an oversight; it now blocks by default like its async counterpart. This is a behaviour change for callers relying on sync unified search against an ingesting index, who should now pass search_during_ingestion=True. The deprecated search and search_async, along with start_deep_rag and start_batch_transform, keep their unconditional guards. Co-Authored-By: Claude Opus 5 --- packages/uipath-platform/pyproject.toml | 2 +- .../_context_grounding_service.py | 22 +++- .../test_context_grounding_service.py | 122 +++++++++++++++++- packages/uipath-platform/uv.lock | 2 +- packages/uipath/pyproject.toml | 4 +- .../_cli/services/cli_context_grounding.py | 8 ++ .../src/uipath/_resources/CLI_REFERENCE.md | 1 + .../src/uipath/_resources/SDK_REFERENCE.md | 4 +- .../uipath/src/uipath/agent/models/agent.py | 9 ++ .../uipath/tests/agent/models/test_agent.py | 47 +++++++ packages/uipath/uv.lock | 4 +- 11 files changed, 215 insertions(+), 10 deletions(-) diff --git a/packages/uipath-platform/pyproject.toml b/packages/uipath-platform/pyproject.toml index d2c8eba69..a674cbce1 100644 --- a/packages/uipath-platform/pyproject.toml +++ b/packages/uipath-platform/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-platform" -version = "0.2.28" +version = "0.2.29" description = "HTTP client library for programmatic access to UiPath Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath-platform/src/uipath/platform/context_grounding/_context_grounding_service.py b/packages/uipath-platform/src/uipath/platform/context_grounding/_context_grounding_service.py index f4cfbe4c9..224a55adc 100644 --- a/packages/uipath-platform/src/uipath/platform/context_grounding/_context_grounding_service.py +++ b/packages/uipath-platform/src/uipath/platform/context_grounding/_context_grounding_service.py @@ -1669,6 +1669,7 @@ def unified_search( folder_key: Optional[str] = None, folder_path: Optional[str] = None, include_system_indexes: bool = False, + search_during_ingestion: bool = False, ) -> UnifiedQueryResult: """Perform a unified search on a context grounding index. @@ -1687,9 +1688,17 @@ def unified_search( include_system_indexes (bool): If True, fall back to tenant-wide system indexes when the index is not found in folder or across-folders listings. Defaults to False. + search_during_ingestion (bool): If True, query the index even while an + ingestion is in progress, returning results from the documents + indexed so far. Defaults to False, which raises + IngestionInProgressException instead. Returns: UnifiedQueryResult: The unified search result containing semantic and/or tabular results. + + Raises: + IngestionInProgressException: If the index is still ingesting and + search_during_ingestion is False. """ index = self.retrieve( name, @@ -1697,6 +1706,8 @@ def unified_search( folder_path=folder_path, include_system_indexes=include_system_indexes, ) + if not search_during_ingestion and index and index.in_progress_ingestion(): + raise IngestionInProgressException(index_name=name) folder_key = folder_key or index.folder_key @@ -1733,6 +1744,7 @@ async def unified_search_async( folder_key: Optional[str] = None, folder_path: Optional[str] = None, include_system_indexes: bool = False, + search_during_ingestion: bool = False, ) -> UnifiedQueryResult: """Asynchronously perform a unified search on a context grounding index. @@ -1751,9 +1763,17 @@ async def unified_search_async( include_system_indexes (bool): If True, fall back to tenant-wide system indexes when the index is not found in folder or across-folders listings. Defaults to False. + search_during_ingestion (bool): If True, query the index even while an + ingestion is in progress, returning results from the documents + indexed so far. Defaults to False, which raises + IngestionInProgressException instead. Returns: UnifiedQueryResult: The unified search result containing semantic and/or tabular results. + + Raises: + IngestionInProgressException: If the index is still ingesting and + search_during_ingestion is False. """ index = await self.retrieve_async( name, @@ -1761,7 +1781,7 @@ async def unified_search_async( folder_path=folder_path, include_system_indexes=include_system_indexes, ) - if index and index.in_progress_ingestion(): + if not search_during_ingestion and index and index.in_progress_ingestion(): raise IngestionInProgressException(index_name=name) folder_key = folder_key or index.folder_key diff --git a/packages/uipath-platform/tests/services/test_context_grounding_service.py b/packages/uipath-platform/tests/services/test_context_grounding_service.py index 2461149d1..30c198153 100644 --- a/packages/uipath-platform/tests/services/test_context_grounding_service.py +++ b/packages/uipath-platform/tests/services/test_context_grounding_service.py @@ -35,7 +35,10 @@ from uipath.platform.context_grounding._context_grounding_service import ( ContextGroundingService, ) -from uipath.platform.errors import ContextGroundingIndexNotFoundError +from uipath.platform.errors import ( + ContextGroundingIndexNotFoundError, + IngestionInProgressException, +) from uipath.platform.orchestrator._buckets_service import BucketsService from uipath.platform.orchestrator._folder_service import FolderService @@ -3839,6 +3842,123 @@ async def test_unified_search_async( assert response.semantic_results.metadata is not None assert len(response.semantic_results.values) == 1 + def _mock_index_lookup( + self, + httpx_mock: HTTPXMock, + base_url: str, + org: str, + tenant: str, + ingestion_status: str, + ) -> None: + """Mock the folder + index lookup that precedes a unified search.""" + for _ in range(2): + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/api/FoldersNavigation/GetFoldersForCurrentUser?searchText=test-folder-path&skip=0&take=20", + status_code=200, + json={ + "PageItems": [ + { + "Key": "test-folder-key", + "FullyQualifiedName": "test-folder-path", + } + ] + }, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes?$filter=Name eq 'test-index'&$expand=dataSource", + status_code=200, + json={ + "value": [ + { + "id": "test-index-id", + "name": "test-index", + "lastIngestionStatus": ingestion_status, + } + ] + }, + ) + + @pytest.mark.anyio + @pytest.mark.httpx_mock(assert_all_responses_were_requested=False) + @pytest.mark.parametrize("ingestion_status", ["Queued", "InProgress"]) + async def test_unified_search_async_blocks_during_ingestion_by_default( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ingestion_status: str, + ) -> None: + self._mock_index_lookup(httpx_mock, base_url, org, tenant, ingestion_status) + + with pytest.raises(IngestionInProgressException): + await service.unified_search_async(name="test-index", query="test query") + + @pytest.mark.httpx_mock(assert_all_responses_were_requested=False) + @pytest.mark.parametrize("ingestion_status", ["Queued", "InProgress"]) + def test_unified_search_blocks_during_ingestion_by_default( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ingestion_status: str, + ) -> None: + self._mock_index_lookup(httpx_mock, base_url, org, tenant, ingestion_status) + + with pytest.raises(IngestionInProgressException): + service.unified_search(name="test-index", query="test query") + + @pytest.mark.anyio + @pytest.mark.parametrize("ingestion_status", ["Queued", "InProgress"]) + async def test_unified_search_async_opt_in_searches_during_ingestion( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ingestion_status: str, + ) -> None: + self._mock_index_lookup(httpx_mock, base_url, org, tenant, ingestion_status) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v1.2/search/test-index-id", + status_code=200, + json={ + "semanticResults": { + "metadata": { + "operation_id": "test-op", + "strategy": "test-strategy", + }, + "values": [ + { + "id": "result-1", + "source": "test-source", + "page_number": 1, + "content": "Partially ingested content", + "score": 0.95, + } + ], + }, + "explanation": "test explanation", + }, + ) + + response = await service.unified_search_async( + name="test-index", + query="test query", + search_during_ingestion=True, + ) + + assert isinstance(response, UnifiedQueryResult) + assert response.semantic_results is not None + assert len(response.semantic_results.values) == 1 + assert ( + response.semantic_results.values[0].content == "Partially ingested content" + ) + def test_unified_search_with_scope( self, httpx_mock: HTTPXMock, diff --git a/packages/uipath-platform/uv.lock b/packages/uipath-platform/uv.lock index 93559e572..183e9462b 100644 --- a/packages/uipath-platform/uv.lock +++ b/packages/uipath-platform/uv.lock @@ -1095,7 +1095,7 @@ dev = [ [[package]] name = "uipath-platform" -version = "0.2.28" +version = "0.2.29" source = { editable = "." } dependencies = [ { name = "anyio" }, diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 0b1b056fc..178c86446 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "uipath" -version = "2.14.13" +version = "2.14.14" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" dependencies = [ "uipath-core>=0.5.30, <0.6.0", "uipath-runtime>=0.13.1, <0.14.0", - "uipath-platform>=0.2.27, <0.3.0", + "uipath-platform>=0.2.29, <0.3.0", "click>=8.3.1", "httpx>=0.28.1", "pyjwt>=2.10.1", diff --git a/packages/uipath/src/uipath/_cli/services/cli_context_grounding.py b/packages/uipath/src/uipath/_cli/services/cli_context_grounding.py index 1e25def5e..a1fbc9f2f 100644 --- a/packages/uipath/src/uipath/_cli/services/cli_context_grounding.py +++ b/packages/uipath/src/uipath/_cli/services/cli_context_grounding.py @@ -464,6 +464,12 @@ def ingest_index( default="Semantic", help="Search mode (default: Semantic)", ) +@click.option( + "--search-during-ingestion", + is_flag=True, + default=False, + help="Search the documents indexed so far even while ingestion is in progress", +) @common_service_options @service_command def search_index( @@ -473,6 +479,7 @@ def search_index( limit: int, threshold: float, search_mode: str, + search_during_ingestion: bool, folder_path: Optional[str], folder_key: Optional[str], format: Optional[str], @@ -494,6 +501,7 @@ def search_index( number_of_results=limit, threshold=threshold, search_mode=SearchMode(search_mode), + search_during_ingestion=search_during_ingestion, folder_path=folder_path, folder_key=folder_key, ) diff --git a/packages/uipath/src/uipath/_resources/CLI_REFERENCE.md b/packages/uipath/src/uipath/_resources/CLI_REFERENCE.md index ff4009b22..48cb9ad10 100644 --- a/packages/uipath/src/uipath/_resources/CLI_REFERENCE.md +++ b/packages/uipath/src/uipath/_resources/CLI_REFERENCE.md @@ -924,6 +924,7 @@ Options: - `--limit`: Maximum number of results (default: 10) (default: `10`) - `--threshold`: Minimum similarity threshold (default: 0.0) (default: `0.0`) - `--search-mode`: Search mode (default: Semantic) (default: `Semantic`) +- `--search-during-ingestion`: Search the documents indexed so far even while ingestion is in progress - `--folder-path`: Folder path (e.g., "Shared"). Can also be set via UIPATH_FOLDER_PATH environment variable. (default: `Sentinel.UNSET`) - `--folder-key`: Folder key (UUID) (default: `Sentinel.UNSET`) - `--format`: Output format (overrides global) (default: `Sentinel.UNSET`) diff --git a/packages/uipath/src/uipath/_resources/SDK_REFERENCE.md b/packages/uipath/src/uipath/_resources/SDK_REFERENCE.md index 02e9c0676..704dfa078 100644 --- a/packages/uipath/src/uipath/_resources/SDK_REFERENCE.md +++ b/packages/uipath/src/uipath/_resources/SDK_REFERENCE.md @@ -405,10 +405,10 @@ sdk.context_grounding.start_deep_rag_ephemeral(name: str, prompt: Annotated[str, sdk.context_grounding.start_deep_rag_ephemeral_async(name: str, prompt: Annotated[str, FieldInfo(annotation=NoneType, required=True, metadata=[MaxLen(max_length=250000)])], glob_pattern: Annotated[str, FieldInfo(annotation=NoneType, required=False, default='*', metadata=[MaxLen(max_length=512)])]="**", citation_mode: uipath.platform.context_grounding.context_grounding.DeepRagCreationResponse # Perform a unified search on a context grounding index. -sdk.context_grounding.unified_search(name: str, query: str, search_mode: uipath.platform.context_grounding.context_grounding.UnifiedQueryResult +sdk.context_grounding.unified_search(name: str, query: str, search_mode: uipath.platform.context_grounding.context_grounding.UnifiedQueryResult # Asynchronously perform a unified search on a context grounding index. -sdk.context_grounding.unified_search_async(name: str, query: str, search_mode: uipath.platform.context_grounding.context_grounding.UnifiedQueryResult +sdk.context_grounding.unified_search_async(name: str, query: str, search_mode: uipath.platform.context_grounding.context_grounding.UnifiedQueryResult ``` diff --git a/packages/uipath/src/uipath/agent/models/agent.py b/packages/uipath/src/uipath/agent/models/agent.py index 73ac969a9..d633fb045 100644 --- a/packages/uipath/src/uipath/agent/models/agent.py +++ b/packages/uipath/src/uipath/agent/models/agent.py @@ -427,6 +427,15 @@ class AgentContextSettings(BaseCfg): output_columns: Optional[List[AgentContextOutputColumn]] = Field( None, alias="outputColumns" ) + search_during_ingestion: bool = Field( + default=False, + alias="searchDuringIngestion", + description=( + "Allow the agent to query this index while an ingestion is still in " + "progress, returning results from the documents indexed so far. When " + "False, a search against an ingesting index fails instead." + ), + ) class AgentContextResourceConfig(BaseAgentResourceConfig): diff --git a/packages/uipath/tests/agent/models/test_agent.py b/packages/uipath/tests/agent/models/test_agent.py index ec0a6351f..bec059bcf 100644 --- a/packages/uipath/tests/agent/models/test_agent.py +++ b/packages/uipath/tests/agent/models/test_agent.py @@ -11,6 +11,7 @@ AgentClientSideToolResourceConfig, AgentContextResourceConfig, AgentContextRetrievalMode, + AgentContextSettings, AgentContextType, AgentCustomGuardrail, AgentDefinition, @@ -4778,3 +4779,49 @@ def test_native_bag_survives_verbatim_by_alias(self): self._agent_settings(modelSettings=native) ) assert settings.model_dump(by_alias=True)["modelSettings"] == native + + +class TestSearchDuringIngestion: + """settings.searchDuringIngestion opts a context resource into mid-ingestion search.""" + + def _context_settings(self, **extra: Any) -> dict[str, Any]: + return { + "threshold": 0, + "resultCount": 3, + "retrievalMode": "Semantic", + "query": {"description": "The query.", "variant": "Dynamic"}, + **extra, + } + + def test_defaults_to_false_when_absent(self): + settings = AgentContextSettings.model_validate(self._context_settings()) + assert settings.search_during_ingestion is False + + @pytest.mark.parametrize( + "key", ["searchDuringIngestion", "search_during_ingestion"] + ) + def test_opt_in_parsed_from_alias_and_field_name(self, key: str): + settings = AgentContextSettings.model_validate( + self._context_settings(**{key: True}) + ) + assert settings.search_during_ingestion is True + + def test_round_trips_by_alias(self): + settings = AgentContextSettings.model_validate( + self._context_settings(searchDuringIngestion=True) + ) + assert settings.model_dump(by_alias=True)["searchDuringIngestion"] is True + + def test_reaches_the_resource_config(self): + resource = AgentContextResourceConfig.model_validate( + { + "$resourceType": "context", + "folderPath": "TestFolder", + "indexName": "Test Index", + "name": "Test Context", + "description": "Context that may be searched mid-ingestion", + "settings": self._context_settings(searchDuringIngestion=True), + } + ) + assert resource.settings is not None + assert resource.settings.search_during_ingestion is True diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 18734c70a..6ecdc608c 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2599,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.14.13" +version = "2.14.14" source = { editable = "." } dependencies = [ { name = "applicationinsights" }, @@ -2762,7 +2762,7 @@ wheels = [ [[package]] name = "uipath-platform" -version = "0.2.28" +version = "0.2.29" source = { editable = "../uipath-platform" } dependencies = [ { name = "anyio" },