UN-1924 [FIX] Reject unsupported files in API deployment

backend/api_v2/deployment_helper.py

@@ -19,6 +19,7 @@
 from utils.constants import Account, CeleryQueue
 from utils.local_context import StateStore
 from workflow_manager.endpoint_v2.destination import DestinationConnector
+from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils
 from workflow_manager.endpoint_v2.source import SourceConnector
 from workflow_manager.workflow_v2.dto import ExecutionResponse
 from workflow_manager.workflow_v2.enums import ExecutionStatus
@@ -293,7 +294,14 @@ def execute_workflow(
                 logger.exception(f"Failed to mark execution {execution_id} as ERROR")
 
             # Async job never started — release the rate limit slot and clean up.
-            APIDeploymentRateLimiter.release_slot(api.organization, str(execution_id))
+            # str(...organization_id), NOT the model instance: release_slot formats
+            # its argument into the Redis key, and acquire_slot built that key from
+            # str(organization.organization_id). Passing the instance ZREMs a
+            # non-member — it returns 0 and raises nothing, so the slot silently
+            # stays held for the full TTL. Same trap as undispatched_sweep.py:245.
+            APIDeploymentRateLimiter.release_slot(
+                str(api.organization.organization_id), str(execution_id)
+            )
             DestinationConnector.delete_api_storage_dir(
                 workflow_id=workflow_id, execution_id=execution_id
             )
@@ -306,6 +314,45 @@ def execute_workflow(
                 )
             ).data
 
+        # Staging rejected every file, so there is nothing to dispatch. The worker
+        # short-circuits an empty file set without writing a status back, which
+        # would strand this execution in PENDING — terminalise it here instead.
+        if not hash_values_of_files:
+            # Isolate the DB write the way the staging-failure path above does, so
+            # the rate limit slot and staging dir are released even if it raises.
+            execution = None
+            try:
+                execution = WorkflowExecutionServiceHelper.update_execution_completed(
+                    str(execution_id),
+                    total_files=len(file_objs),
+                    failed_files=len(file_objs),
+                )
+            except Exception:
+                logger.exception(f"Failed to mark execution {execution_id} as COMPLETED")
+
+            APIDeploymentRateLimiter.release_slot(
+                str(api.organization.organization_id), str(execution_id)
+            )
+            DestinationConnector.delete_api_storage_dir(
+                workflow_id=workflow_id, execution_id=execution_id
+            )
+            # Report the stored status rather than asserting COMPLETED: the row may
+            # be missing, or the terminal guard may have refused the change. Claiming
+            # success here would only hide the stranded execution behind a 200 that a
+            # follow-up GET /status then contradicts.
+            return APIExecutionResponseSerializer(
+                ExecutionResponse(
+                    workflow_id=workflow_id,
+                    execution_id=execution_id,
+                    execution_status=(
+                        execution.status if execution else ExecutionStatus.ERROR.value
+                    ),
+                    result=ResultCacheUtils.get_api_results(
+                        workflow_id=str(workflow_id), execution_id=str(execution_id)
+                    ),
+                )
+            ).data
+
         try:
             result = WorkflowHelper.execute_workflow_async(
                 workflow_id=workflow_id,
@@ -352,7 +399,9 @@ def execute_workflow(
             # Dispatch failures are marked ERROR internally by execute_workflow_async;
             # post-dispatch failures (enrichment/config) must not overwrite a running
             # execution's status, so only release the slot and clean up storage here.
-            APIDeploymentRateLimiter.release_slot(api.organization, str(execution_id))
+            APIDeploymentRateLimiter.release_slot(
+                str(api.organization.organization_id), str(execution_id)
+            )
 
             # Clean up storage
             DestinationConnector.delete_api_storage_dir(

backend/api_v2/tests/test_deployment_helper.py

@@ -38,8 +38,8 @@ def collaborators():
         mocks[
             "WorkflowExecutionServiceHelper"
         ].create_workflow_execution.return_value = execution_row
-        mocks["SourceConnector"].add_input_file_to_api_storage.side_effect = (
-            RuntimeError("boom")
+        mocks["SourceConnector"].add_input_file_to_api_storage.side_effect = RuntimeError(
+            "boom"
         )
         yield mocks
 
@@ -48,6 +48,7 @@ def _api() -> MagicMock:
     api = MagicMock()
     api.workflow.id = "wf-1"
     api.id = "pipe-1"
+    api.organization.organization_id = "org-uuid-1"
     return api
 
 
@@ -74,9 +75,9 @@ def test_staging_failure_marks_execution_error(collaborators) -> None:
 
 def test_staging_failure_cleanup_survives_db_marking_error(collaborators) -> None:
     """If marking the row ERROR itself raises, cleanup must still run (not propagate)."""
-    collaborators["WorkflowExecutionServiceHelper"].update_execution_err.side_effect = (
-        RuntimeError("db down")
-    )
+    collaborators[
+        "WorkflowExecutionServiceHelper"
+    ].update_execution_err.side_effect = RuntimeError("db down")
 
     # Must NOT raise — a failed error-marking should not break cleanup.
     dh.DeploymentHelper.execute_workflow(
@@ -89,3 +90,124 @@ def test_staging_failure_cleanup_survives_db_marking_error(collaborators) -> Non
     # Cleanup still runs even though error-marking raised.
     collaborators["APIDeploymentRateLimiter"].release_slot.assert_called_once()
     collaborators["DestinationConnector"].delete_api_storage_dir.assert_called_once()
+
+
+@pytest.fixture
+def staging_rejects_everything():
+    """Patch execute_workflow's collaborators; staging returns no dispatchable files."""
+    with mock.patch.multiple(
+        dh,
+        WorkflowExecutionServiceHelper=mock.DEFAULT,
+        SourceConnector=mock.DEFAULT,
+        DestinationConnector=mock.DEFAULT,
+        APIDeploymentRateLimiter=mock.DEFAULT,
+        WorkflowHelper=mock.DEFAULT,
+        ResultCacheUtils=mock.DEFAULT,
+        Tag=mock.DEFAULT,
+        logger=mock.DEFAULT,
+    ) as mocks:
+        execution_row = MagicMock()
+        execution_row.id = "exec-123"
+        mocks[
+            "WorkflowExecutionServiceHelper"
+        ].create_workflow_execution.return_value = execution_row
+        mocks["SourceConnector"].add_input_file_to_api_storage.return_value = {}
+        mocks["ResultCacheUtils"].get_api_results.return_value = [
+            {"file": "evil.pdf", "status": "Failed", "error": "unsupported MIME type"}
+        ]
+        completed_row = MagicMock()
+        completed_row.status = "COMPLETED"
+        mocks[
+            "WorkflowExecutionServiceHelper"
+        ].update_execution_completed.return_value = completed_row
+        yield mocks
+
+
+def test_all_files_rejected_completes_without_dispatch(
+    staging_rejects_everything,
+) -> None:
+    """A request whose every file is rejected must reach a terminal status.
+
+    The worker short-circuits an empty file set without writing a status back, so
+    dispatching one strands the execution in PENDING and the caller polls forever.
+    """
+    mocks = staging_rejects_everything
+    # A non-empty upload whose staging result is empty. Passing [] instead would
+    # leave the branch satisfied by `not file_objs` too, and the original bug -
+    # dispatching a request whose files were all rejected - would pass this test.
+    response = dh.DeploymentHelper.execute_workflow(
+        organization_name="org",
+        api=_api(),
+        file_objs=[MagicMock()],
+        timeout=-1,
+    )
+
+    # Nothing is dispatched...
+    mocks["WorkflowHelper"].execute_workflow_async.assert_not_called()
+    # ...the row is terminalised here instead of being left PENDING, and the
+    # counters are written so the run does not read back as a clean success...
+    mocks[
+        "WorkflowExecutionServiceHelper"
+    ].update_execution_completed.assert_called_once_with(
+        "exec-123", total_files=1, failed_files=1
+    )
+    # ...the slot and staging dir are released. The slot must be released by org
+    # id string: release_slot formats its argument into the Redis key, so passing
+    # the model instance removes a non-member and silently holds the slot.
+    mocks["APIDeploymentRateLimiter"].release_slot.assert_called_once_with(
+        "org-uuid-1", "exec-123"
+    )
+    mocks["DestinationConnector"].delete_api_storage_dir.assert_called_once()
+    # ...and the caller still sees why each file failed.
+    assert response["execution_status"] == "COMPLETED"
+    assert response["result"][0]["file"] == "evil.pdf"
+    assert response["result"][0]["status"] == "Failed"
+
+
+def test_files_staged_successfully_are_dispatched(staging_rejects_everything) -> None:
+    """The short-circuit must not fire when staging did return files.
+
+    Sibling to the test above: together they pin the branch to the staging result
+    rather than to the upload list.
+    """
+    mocks = staging_rejects_everything
+    mocks["SourceConnector"].add_input_file_to_api_storage.return_value = {
+        "good.pdf": MagicMock()
+    }
+
+    dh.DeploymentHelper.execute_workflow(
+        organization_name="org",
+        api=_api(),
+        file_objs=[MagicMock()],
+        timeout=-1,
+    )
+
+    mocks["WorkflowHelper"].execute_workflow_async.assert_called_once()
+    mocks["WorkflowExecutionServiceHelper"].update_execution_completed.assert_not_called()
+
+
+def test_all_files_rejected_cleanup_survives_db_marking_error(
+    staging_rejects_everything,
+) -> None:
+    """A failing status write must not strand the slot or the staging dir.
+
+    update_execution_completed only catches DoesNotExist, so a lock timeout or a
+    dropped connection propagates; without isolation the org's rate limit slot
+    stays held for its full TTL and throttles every other call for that org.
+    """
+    mocks = staging_rejects_everything
+    mocks[
+        "WorkflowExecutionServiceHelper"
+    ].update_execution_completed.side_effect = Exception("db is down")
+
+    response = dh.DeploymentHelper.execute_workflow(
+        organization_name="org",
+        api=_api(),
+        file_objs=[MagicMock()],
+        timeout=-1,
+    )
+
+    mocks["APIDeploymentRateLimiter"].release_slot.assert_called_once()
+    mocks["DestinationConnector"].delete_api_storage_dir.assert_called_once()
+    # The row never reached COMPLETED, so the response must not claim it did.
+    assert response["execution_status"] == "ERROR"

backend/workflow_manager/endpoint_v2/source.py

@@ -2,7 +2,6 @@
 import logging
 import os
 import shutil
-import uuid
 from collections.abc import Collection
 from hashlib import sha256
 from io import BytesIO
@@ -25,7 +24,11 @@
     SourceConstant,
     SourceKey,
 )
-from workflow_manager.endpoint_v2.dto import FileHash, SourceConfig
+from workflow_manager.endpoint_v2.dto import (
+    FileExecutionResult,
+    FileHash,
+    SourceConfig,
+)
 from workflow_manager.endpoint_v2.enums import AllowedFileTypes
 from workflow_manager.endpoint_v2.exceptions import (
     InvalidInputDirectory,
@@ -37,6 +40,7 @@
     UnsupportedMimeTypeError,
 )
 from workflow_manager.endpoint_v2.models import WorkflowEndpoint
+from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils
 from workflow_manager.file_execution.models import WorkflowFileExecution
 from workflow_manager.utils.workflow_log import WorkflowLog
 from workflow_manager.workflow_v2.enums import ExecutionStatus
@@ -69,6 +73,15 @@ class SourceConnector(BaseConnector):
     """
 
     READ_CHUNK_SIZE = 4194304  # Chunk size for reading files
+    # Most formats are identifiable from their leading bytes, so a small sample
+    # keeps the common path cheap.
+    MIME_DETECT_CHUNK_SIZE = 8192
+    # These two carry the real format in a structure libmagic can only reach by
+    # reading the whole file: the OLE2 directory sector and the zip central
+    # directory both sit at the end. A sample of any size reports the container
+    # rather than the .doc/.xls/.ppt or .docx/.xlsx/.pptx inside it, so these
+    # must never be resolved from the sample alone.
+    CONTAINER_MIME_TYPES = frozenset({"application/x-ole-storage", "application/zip"})
 
     def __init__(
         self,
@@ -1187,6 +1200,44 @@ def load_file(self, input_file_path: str) -> tuple[str, BytesIO]:
 
         return os.path.basename(input_file_path), file_stream
 
+    @classmethod
+    def _detect_uploaded_file_mime_type(cls, file: UploadedFile) -> str:
+        """Detect an uploaded file's MIME type from its own bytes.
+
+        The multipart Content-Type is supplied by the caller and never verified,
+        so it cannot be used to decide what is allowed into API storage.
+        """
+        sample = file.read(cls.MIME_DETECT_CHUNK_SIZE)
+        file.seek(0)
+        if not sample:
+            # libmagic reports "application/x-empty" here, which would reject the
+            # file as an unsupported type. An empty upload is a distinct failure
+            # and is reported as such once staging hands off, so let it pass.
+            return AllowedFileTypes.OCTET_STREAM.value
+
+        mime_type = magic.from_buffer(sample, mime=True)
+        if mime_type not in cls.CONTAINER_MIME_TYPES:
+            return mime_type
+        return cls._detect_container_mime_type(file, fallback=mime_type)
+
+    @classmethod
+    def _detect_container_mime_type(cls, file: UploadedFile, fallback: str) -> str:
+        """Resolve a container format by classifying the file in full.
+
+        Django spills uploads over FILE_UPLOAD_MAX_MEMORY_SIZE to disk, so this
+        hands libmagic the path when there is one and only buffers the whole
+        upload for the in-memory case, where that ceiling already bounds it.
+        """
+        temporary_file_path = getattr(file, "temporary_file_path", None)
+        if temporary_file_path is not None:
+            return magic.from_file(temporary_file_path(), mime=True)
+
+        content = file.read()
+        file.seek(0)
+        if not content:
+            return fallback
+        return magic.from_buffer(content, mime=True)
+
     @classmethod
     def add_input_file_to_api_storage(
         cls,
@@ -1228,30 +1279,43 @@ def add_input_file_to_api_storage(
             file_name = file.name
             destination_path = os.path.join(api_storage_dir, file_name)
 
-            mime_type = file.content_type
-            logger.info(f"Detected MIME type: {mime_type} for file {file_name}")
-            if not mime_type:
-                logger.info(
-                    f"MIME type not found for file {file_name}, using default MIME type: {AllowedFileTypes.OCTET_STREAM.value}"
+            try:
+                mime_type = cls._detect_uploaded_file_mime_type(file)
+            except Exception:
+                # Detection reads the upload, so a broken stream raises here. Fail
+                # this one file instead of the whole request, and say that detection
+                # failed rather than blaming the file's type - an I/O fault and an
+                # unsupported format need different follow-ups.
+                log_message = (
+                    f"Rejecting file '{file_name}': could not determine its type"
                 )
-                mime_type = AllowedFileTypes.OCTET_STREAM.value
+                logger.exception(log_message)
+                workflow_log.log_error(logger=logger, message=log_message)
+                ResultCacheUtils.update_api_results(
+                    workflow_id=workflow_id,
+                    execution_id=execution_id,
+                    api_result=FileExecutionResult(file=file_name, error=log_message),
+                )
+                continue
+
+            logger.info(f"Detected MIME type: {mime_type} for file {file_name}")
 
             if not AllowedFileTypes.is_allowed(mime_type):
-                log_message = f"Skipping file '{file_name}' to stage due to unsupported MIME type '{mime_type}'"
-                workflow_log.log_info(logger=logger, message=log_message)
-                # Generate a clearly marked temporary hash to avoid reading the file content
-                # Helps to prevent duplicate entries in file executions
-                fake_hash = f"temp-hash-{uuid.uuid4().hex}"
-                file_hash = FileHash(
-                    file_path=destination_path,
-                    source_connection_type=connection_type,
-                    file_name=file_name,
-                    file_hash=fake_hash,
-                    is_executed=True,
-                    file_size=file.size,
-                    mime_type=mime_type,
+                log_message = (
+                    f"Rejecting file '{file_name}' with unsupported MIME type "
+                    f"'{mime_type}'"
+                )
+                workflow_log.log_error(logger=logger, message=log_message)
+                # Rejected files are never dispatched, so nothing downstream will
+                # report on them - surface the failure in the API response here.
+                ResultCacheUtils.update_api_results(
+                    workflow_id=workflow_id,
+                    execution_id=execution_id,
+                    api_result=FileExecutionResult(
+                        file=file_name,
+                        error=log_message,
+                    ),
                 )
-                file_hashes.update({file_name: file_hash})
                 continue
 
             file_system = FileSystem(FileStorageType.API_EXECUTION)