Sprint 4 [FIX] Backend fixes: UN-2900, UN-2902, UN-2953, UN-3038, UN-3133, UN-3176, UN-3333

backend/prompt_studio/prompt_studio_core_v2/prompt_variable_service.py

@@ -64,6 +64,41 @@ def identify_variable_type(variable: str) -> VariableType:
                 variable_type = VariableType.STATIC
         return variable_type
 
+    @staticmethod
+    def find_unresolvable_single_pass_variables(prompt: str) -> list[str]:
+        """Variables in ``prompt`` that cannot resolve under single-pass extraction.
+
+        UN-2900. Single pass builds ONE combined prompt — every field is declared
+        up front in a single JSON schema and answered in one LLM call — so no
+        prompt's output exists to feed another prompt's variable. The runtime
+        reflects this: the enterprise ``single_pass_extraction`` plugin calls the
+        shared replacement service with ``structured_output={}``, and both
+        ``replace_static_variable`` and ``replace_dynamic_variable`` return the
+        prompt UNCHANGED when their lookup misses. The literal ``{{...}}`` is then
+        sent to the LLM, silently degrading the answer.
+
+        CUSTOM_DATA is the one exception: it is resolved from the tool's own
+        ``custom_data`` and never consults the variable map, so it works
+        identically in both modes and is not reported here.
+
+        Returns the offending variable strings, in prompt order, or an empty list.
+
+        NOTE: classification pairs with ``VariableReplacementService`` in the
+        worker (``executor/executors/variable_replacement.py``), which keeps its
+        own copy of these regexes. If one side's patterns change, this validation
+        and the runtime behaviour will disagree.
+        """
+        unresolvable: list[str] = []
+        for variable in PromptStudioVariableService.extract_variables_from_prompt(
+            prompt=prompt
+        ):
+            variable_type = PromptStudioVariableService.identify_variable_type(
+                variable=variable
+            )
+            if variable_type != VariableType.CUSTOM_DATA:
+                unresolvable.append(variable)
+        return unresolvable
+
     @staticmethod
     def extract_variables_from_prompt(prompt: str) -> list[str]:
         variable: list[str] = []

backend/prompt_studio/prompt_studio_core_v2/serializers.py

@@ -186,9 +186,15 @@ def to_representation(self, instance):  # type: ignore
             )
 
         # Fetch prompt instances
-        prompt_instances: ToolStudioPrompt = ToolStudioPrompt.objects.filter(
-            tool_id=data.get(TSKeys.TOOL_ID)
-        ).order_by("sequence_number")
+        # select_related("tool_id"): ToolStudioPromptSerializer's
+        # single_pass_unresolvable_variables (UN-2900) reads the parent tool's
+        # single_pass_extraction_mode, which would otherwise be one query per
+        # prompt here.
+        prompt_instances: ToolStudioPrompt = (
+            ToolStudioPrompt.objects.filter(tool_id=data.get(TSKeys.TOOL_ID))
+            .select_related("tool_id")
+            .order_by("sequence_number")
+        )
 
         data["created_by_email"] = (
             instance.created_by.email if instance.created_by else ""

backend/prompt_studio/prompt_studio_v2/serializers.py

@@ -23,13 +23,39 @@ class Meta:
 
 
 class ToolStudioPromptSerializer(AuditSerializer):
+    single_pass_unresolvable_variables = serializers.SerializerMethodField()
+
     class Meta:
         model = ToolStudioPrompt
         fields = "__all__"
         # View owns uniqueness (IntegrityError->DuplicateData on create); drop
         # the DRF auto-validator that 400s on re-save / PUT before the view runs.
         validators = []
 
+    def get_single_pass_unresolvable_variables(self, obj) -> list[str]:
+        """UN-2900: variables in this prompt that single pass cannot resolve.
+
+        Empty unless the parent tool has single-pass extraction enabled. Lives
+        on this serializer rather than the tool serializer so the same value is
+        returned by BOTH the tool detail fetch (which nests this serializer per
+        prompt) and the prompt save response, without duplicating the logic.
+
+        ``tool_id`` is the FK to CustomTool; callers that serialize many prompts
+        should ``select_related("tool_id")`` to avoid a query per prompt.
+        """
+        from prompt_studio.prompt_studio_core_v2.prompt_variable_service import (
+            PromptStudioVariableService,
+        )
+
+        tool = getattr(obj, "tool_id", None)
+        if not tool or not getattr(tool, "single_pass_extraction_mode", False):
+            return []
+        if not obj.prompt:
+            return []
+        return PromptStudioVariableService.find_unresolvable_single_pass_variables(
+            prompt=obj.prompt
+        )
+
 
 class ToolStudioIndexSerializer(serializers.Serializer):
     file_name = serializers.CharField()

backend/tool_instance_v2/tool_instance_helper.py

@@ -486,33 +486,57 @@ def validate_adapter_permissions(
         adapter_ids: set[str] = set()
 
         for llm in tool.properties.adapter.language_models:
+            adapter_id = None
             if llm.is_enabled and llm.adapter_id:
-                adapter_id = tool_meta[llm.adapter_id]
+                adapter_id = tool_meta.get(llm.adapter_id)
             elif llm.is_enabled:
-                adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_LLM_ADAPTER_ID]
+                adapter_id = tool_meta.get(AdapterPropertyKey.DEFAULT_LLM_ADAPTER_ID)
 
-            adapter_ids.add(adapter_id)
+            # UN-2953: a tool instance may carry "" for an adapter id.
+            # Adding it made the schema validator compare "" against the
+            # UUID enum and raise, once per validation pass.
+            if adapter_id:
+                adapter_ids.add(adapter_id)
         for vdb in tool.properties.adapter.vector_stores:
+            adapter_id = None
             if vdb.is_enabled and vdb.adapter_id:
-                adapter_id = tool_meta[vdb.adapter_id]
+                adapter_id = tool_meta.get(vdb.adapter_id)
             elif vdb.is_enabled:
-                adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_VECTOR_DB_ADAPTER_ID]
+                adapter_id = tool_meta.get(
+                    AdapterPropertyKey.DEFAULT_VECTOR_DB_ADAPTER_ID
+                )
 
-            adapter_ids.add(adapter_id)
+            # UN-2953: a tool instance may carry "" for an adapter id.
+            # Adding it made the schema validator compare "" against the
+            # UUID enum and raise, once per validation pass.
+            if adapter_id:
+                adapter_ids.add(adapter_id)
         for embedding in tool.properties.adapter.embedding_services:
+            adapter_id = None
             if embedding.is_enabled and embedding.adapter_id:
-                adapter_id = tool_meta[embedding.adapter_id]
+                adapter_id = tool_meta.get(embedding.adapter_id)
             elif embedding.is_enabled:
-                adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_EMBEDDING_ADAPTER_ID]
+                adapter_id = tool_meta.get(
+                    AdapterPropertyKey.DEFAULT_EMBEDDING_ADAPTER_ID
+                )
 
-            adapter_ids.add(adapter_id)
+            # UN-2953: a tool instance may carry "" for an adapter id.
+            # Adding it made the schema validator compare "" against the
+            # UUID enum and raise, once per validation pass.
+            if adapter_id:
+                adapter_ids.add(adapter_id)
         for text_extractor in tool.properties.adapter.text_extractors:
+            adapter_id = None
             if text_extractor.is_enabled and text_extractor.adapter_id:
-                adapter_id = tool_meta[text_extractor.adapter_id]
+                adapter_id = tool_meta.get(text_extractor.adapter_id)
             elif text_extractor.is_enabled:
-                adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_X2TEXT_ADAPTER_ID]
+                adapter_id = tool_meta.get(AdapterPropertyKey.DEFAULT_X2TEXT_ADAPTER_ID)
 
-            adapter_ids.add(adapter_id)
+            # UN-2953: a tool instance may carry "" for an adapter id.
+            # Adding it made the schema validator compare "" against the
+            # UUID enum and raise, once per validation pass.
+            if adapter_id:
+                adapter_ids.add(adapter_id)
 
         ToolInstanceHelper.validate_adapter_access(user=user, adapter_ids=adapter_ids)
 

unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py

@@ -13,6 +13,7 @@
 from unstract.connectors.databases.exceptions import (
     BigQueryForbiddenException,
     BigQueryNotFoundException,
+    BigQueryValueException,
     ColumnMissingException,
 )
 from unstract.connectors.databases.sql_safety import (
@@ -106,13 +107,14 @@
             if data == 0:
                 return 0.0
 
-            # Limit total significant figures to 15 for IEEE 754 compatibility
-            # BigQuery PARSE_JSON requires values that round-trip cleanly
-            # For large numbers (like Unix timestamps), this reduces decimal precision
-            # For small numbers (like costs), full precision is preserved
-            magnitude = math.floor(math.log10(abs(data))) + 1
-            safe_decimals = max(0, 15 - magnitude)
-            return float(f"{data:.{safe_decimals}f}")
+            # Limit total significant figures to 15 for IEEE 754 compatibility.
+            # BigQuery PARSE_JSON requires values that round-trip cleanly.
+            #
+            # UN-3176: `g` asks for significant figures directly. The previous
+            # form asked for a DECIMAL-place count derived from the magnitude,
+            # which is not the same quantity and drifted from 15 sig figs at
+            # both ends of the range.
+            return float(f"{data:.15g}")
 
         elif isinstance(data, dict):
             return {k: BigQuery._sanitize_for_bigquery(v) for k, v in data.items()}
@@ -318,6 +320,17 @@
                 detail=e.message, table_name=table_name
             ) from e
         except google.api_core.exceptions.BadRequest as e:
+            # UN-3176: BigQuery returns BadRequest for VALUE-level failures as
+            # well as for schema mismatches. Mapping them all to
+            # ColumnMissingException told users to check their columns when the
+            # columns were fine (e.g. a float that will not round-trip through
+            # PARSE_JSON), which misdirects the investigation. Discriminate
+            # before wrapping.
+            if BigQuery._is_value_error(e):
+                logger.error(f"Value rejected by BigQuery on insert: {str(e)}")
+                raise BigQueryValueException(
+                    detail=e.message, table_name=table_name
+                ) from e
             logger.error(f"Column missing in inserting data: {str(e)}")
             db, schema, table = table_name.split(".")
             raise ColumnMissingException(
@@ -327,6 +340,38 @@
                 table_name=table,
             ) from e
 
+    @staticmethod
+    def _is_value_error(e: Any) -> bool:
+        """True if a BigQuery BadRequest is about the DATA, not the schema.
+
+        UN-3176. Matches known value-level signatures -- notably the PARSE_JSON
+        round-trip failure in this ticket -- against the exception's own text
+        and then against each entry of its structured ``errors`` payload. The
+        payload is checked separately because BigQuery's REST path fills it
+        with plain dicts, and ``GoogleAPICallError.__str__`` folds an entry
+        into the string only when it exposes ``.code``/``.message`` attributes.
+        Unknown shapes fall through to the existing column-missing behaviour,
+        so this only ever narrows a message that was already wrong for these
+        cases.
+        """
+        value_error_markers = (
+            "parse_json",
+            "round-trip through string representation",
+            "invalid json",
+            "cannot round-trip",
+            "failed to parse json",
+        )
+        text = f"{getattr(e, 'message', '') or ''} {str(e)}".lower()
+        if any(marker in text for marker in value_error_markers):
+            return True
+        for error in getattr(e, "errors", None) or []:
+            if not isinstance(error, dict):
+                continue
+            message = str(error.get("message", "")).lower()
+            if any(marker in message for marker in value_error_markers):
+                return True
+        return False
+
     def get_information_schema(self, table_name: str) -> dict[str, str]:
         """Function to generate information schema of the big query table.
 

unstract/connectors/src/unstract/connectors/databases/exceptions.py

@@ -90,6 +90,29 @@ def __init__(self, detail: str, table_name: str) -> None:
         super().__init__(detail=final_detail)
 
 
+class BigQueryValueException(UnstractDBConnectorException):
+    """A BigQuery BadRequest caused by the DATA, not the table schema.
+
+    UN-3176: BigQuery returns BadRequest for value-level failures (a float that
+    will not round-trip through PARSE_JSON, a malformed JSON literal) as well as
+    for genuine schema mismatches. Mapping every BadRequest to
+    ColumnMissingException told users to "make sure all the columns exist" when
+    the columns were fine, which sent at least one investigation down the wrong
+    path.
+    """
+
+    def __init__(self, detail: Any, table_name: str) -> None:
+        default_detail = (
+            f"Error writing to '{table_name}'. \n"
+            f"BigQuery rejected a value in the row being inserted -- the table "
+            f"schema is not the problem. This usually means a number could not "
+            f"be represented exactly, or a JSON column received text that is "
+            f"not valid JSON.\n"
+        )
+        final_detail = _format_exception_detail(default_detail, detail)
+        super().__init__(detail=final_detail)
+
+
 class ColumnMissingException(UnstractDBConnectorException):
     def __init__(
         self,