diff --git a/ai/react_agent/ocr.py b/ai/react_agent/ocr.py index 76d0139..8f047f7 100644 --- a/ai/react_agent/ocr.py +++ b/ai/react_agent/ocr.py @@ -1,7 +1,10 @@ -import httpx import base64 import os +import httpx + +from ai.prompts import OCR_PROMPT + DASHSCOPE_API_KEY = os.environ.get("DASHSCOPE_API_KEY", "") QWEN_API_BASE = os.environ.get( "QWEN_API_BASE", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" @@ -10,11 +13,53 @@ # Dedicated OCR model — faster and more accurate for text extraction QWEN_OCR_MODEL = os.environ.get("QWEN_OCR_MODEL", "qwen-vl-ocr") -from ai.prompts import OCR_PROMPT + +class OcrError(Exception): + """Raised when OCR can't produce usable text from an image. + + Covers a malformed provider response (missing `choices`/`content`) and a + blank result (an image-only scan with nothing readable). The backend maps + this to a friendly `EXTRACTION_FAILED` instead of crashing with a raw + KeyError/IndexError (which previously surfaced as a 500). + """ + + +def _parse_ocr_response(result: object) -> str: + """Defensively pull the text content out of a chat-completions response. + + The happy path is `result["choices"][0]["message"]["content"]`, but blank + scans and provider hiccups can return no choices, a null message, or null + content. Any of those previously raised KeyError/IndexError/TypeError and + bubbled up as a 500 — here they become a typed `OcrError`. + """ + if not isinstance(result, dict): + raise OcrError("The document reader returned an unexpected response.") + + choices = result.get("choices") + if not isinstance(choices, list) or not choices: + raise OcrError( + "Could not read any text from this document. It may be a scanned " + "image without readable content." + ) + + message = (choices[0] or {}).get("message") if isinstance(choices[0], dict) else None + content = (message or {}).get("content") if isinstance(message, dict) else None + + if not content or not str(content).strip(): + raise OcrError( + "Could not read any text from this document. It may be a scanned " + "image without readable content." + ) + + return str(content) async def extract_text_from_image(image_path: str) -> str: - """Send image to Qwen-VL-OCR and return extracted text.""" + """Send image to Qwen-VL-OCR and return extracted text. + + Raises `OcrError` if the provider returns a malformed or empty response + (e.g. a blank/unreadable scan) so the caller can surface a graceful error. + """ with open(image_path, "rb") as f: image_bytes = f.read() @@ -53,5 +98,4 @@ async def extract_text_from_image(image_path: str) -> str: }, ) response.raise_for_status() - result = response.json() - return result["choices"][0]["message"]["content"] + return _parse_ocr_response(response.json()) diff --git a/backend/app/pipeline/orchestrator.py b/backend/app/pipeline/orchestrator.py index f0436ce..6048a0a 100644 --- a/backend/app/pipeline/orchestrator.py +++ b/backend/app/pipeline/orchestrator.py @@ -39,6 +39,7 @@ import logging import os import re +import tempfile from datetime import date, datetime from typing import AsyncIterator from uuid import UUID @@ -56,11 +57,15 @@ ) from app.services import ai_bridge from app.services.amounts import primary_outstanding_amount -from app.services.extraction import normalize_lang +from app.services.extraction import ExtractionError, normalize_lang +from app.services.pdf_pages import PdfRenderError, pdf_to_image_bytes from app.services.risk import compute_risk logger = logging.getLogger("klar.pipeline") +# Cap how many PDF pages we OCR — bounds cost/latency for very long documents. +_MAX_OCR_PAGES = 5 + # ---------- helpers ---------- @@ -93,6 +98,68 @@ def _mark_error(letter_id: UUID, message: str) -> None: pass +async def _ocr_letter_file(path: str) -> str: + """OCR any uploaded letter file, transparently handling PDFs. + + The AI team's `extract_text_from_image` only understands image files — it + base64-encodes the raw bytes and ships them to the image OCR model. For a + PDF that means sending raw `%PDF` bytes labelled `image/jpeg`, which never + yields text (and for a scanned, image-only PDF used to crash the pipeline). + + Here we render PDFs to one PNG per page (via poppler/pdf2image), OCR each + page, and concatenate the text. Non-PDF files are OCR'd directly. + + Raises: + PdfRenderError: the PDF couldn't be rendered (corrupt / poppler missing). + ExtractionError: OCR produced no readable text (e.g. a blank scan). + """ + from ai.react_agent.ocr import OcrError, extract_text_from_image + + is_pdf = path.lower().endswith(".pdf") + + if not is_pdf: + try: + text = await extract_text_from_image(path) + except OcrError as exc: + raise ExtractionError(str(exc)) from exc + if not text or not text.strip(): + raise ExtractionError( + "Could not read any text from this document. It may be a scanned " + "image without readable content." + ) + return text + + # PDF: render pages to PNGs, OCR each, then join. PdfRenderError propagates. + page_images = pdf_to_image_bytes(path, max_pages=_MAX_OCR_PAGES) + + page_texts: list[str] = [] + for index, png_bytes in enumerate(page_images): + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp: + tmp.write(png_bytes) + tmp_path = tmp.name + try: + page_text = await extract_text_from_image(tmp_path) + except OcrError: + # One unreadable page shouldn't sink a multi-page document — skip it. + logger.info("OCR returned no text for PDF page %d of %s", index + 1, path) + page_text = "" + finally: + try: + os.unlink(tmp_path) + except OSError: + pass + if page_text and page_text.strip(): + page_texts.append(page_text.strip()) + + combined = "\n\n".join(page_texts).strip() + if not combined: + raise ExtractionError( + "Could not extract text from this PDF. It may be a scanned image " + "without readable content." + ) + return combined + + # ---------- German-date regex fallback ---------- # The AI agent occasionally fails to surface an explicit deadline even when # the OCR text plainly contains one (especially for past dates). We scan the @@ -194,7 +261,10 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # BEFORE its first yield — that race causes the browser to see # ERR_INCOMPLETE_CHUNKED_ENCODING and infinitely reconnect. try: - from ai.react_agent.ocr import extract_text_from_image + # OCR is invoked via `_ocr_letter_file` (which lazily imports the OCR + # module); we still import the agent here so a missing TAVILY_API_KEY / + # Tavily-tool instantiation failure surfaces as an SSE error before the + # first yield rather than mid-stream. from ai.react_agent.agent import run_react_agent except Exception as exc: logger.exception( @@ -231,8 +301,10 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str try: # ============================================================ # STAGE 1 — OCR (qwen-vl-ocr, ~3s) + # PDFs are rendered to page images first; blank/unreadable scans + # raise a typed error mapped to a friendly SSE event below. # ============================================================ - ocr_text = await extract_text_from_image(letter.original_file) + ocr_text = await _ocr_letter_file(letter.original_file) letter.ocr_text = ocr_text db.add(letter) db.commit() @@ -454,6 +526,35 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str {"letter_id": str(letter.id), "letter": public.model_dump(mode="json")}, ) + except PdfRenderError as exc: + # PDF couldn't be rendered to images (corrupt / poppler missing). + logger.warning("PDF render failed for letter %s: %s", letter_id, exc) + try: + letter.status = LetterStatus.ERROR + db.add(letter) + db.commit() + except Exception: + _mark_error(letter_id, str(exc)) + yield sse_event( + "error", + sse_error_payload(ErrorCode.PDF_RENDER_FAILED, message=str(exc)), + ) + + except ExtractionError as exc: + # Scanned image-only PDF (no text layer) or blank/unreadable scan — + # surface the typed, user-friendly message instead of a raw error. + logger.info("Extraction produced no text for letter %s: %s", letter_id, exc) + try: + letter.status = LetterStatus.ERROR + db.add(letter) + db.commit() + except Exception: + _mark_error(letter_id, str(exc)) + yield sse_event( + "error", + sse_error_payload(ErrorCode.EXTRACTION_FAILED, message=str(exc)), + ) + except Exception as exc: # noqa: BLE001 — last-resort SSE error event logger.exception("Pipeline failed for letter %s: %s", letter_id, exc) try: diff --git a/backend/app/routers/letters.py b/backend/app/routers/letters.py index 11c8d54..4812bbb 100644 --- a/backend/app/routers/letters.py +++ b/backend/app/routers/letters.py @@ -18,7 +18,12 @@ utcnow, ) from app.schemas import ErrorResponse, LetterListItem, LetterResponse, LetterUploadResponse -from app.services.extraction import extract_from_letter_file, normalize_lang +from app.services.extraction import ( + ExtractionError, + extract_from_letter_file, + normalize_lang, +) +from app.services.pdf_pages import PdfRenderError from app.services.persistence import persist_extraction from app.services.storage import detect_magic_mime, save_letter_file @@ -215,6 +220,20 @@ async def extract_letter( extracted = await extract_from_letter_file( letter.original_file, mime, lang=letter.language ) + except PdfRenderError as exc: + letter.status = LetterStatus.ERROR + db.add(letter) + db.commit() + # PDF couldn't be rendered (corrupt / poppler missing) — distinct, + # actionable message ("try uploading it as an image instead"). + raise KlarHTTPException(502, ErrorCode.PDF_RENDER_FAILED, message=str(exc)) + except ExtractionError as exc: + letter.status = LetterStatus.ERROR + db.add(letter) + db.commit() + # Scanned image-only PDF (no text layer) or malformed model output — + # surface the typed, user-friendly message instead of a raw 500. + raise KlarHTTPException(502, ErrorCode.EXTRACTION_FAILED, message=str(exc)) except Exception: letter.status = LetterStatus.ERROR db.add(letter) diff --git a/backend/app/routers/public.py b/backend/app/routers/public.py index 26c9bd1..4b03459 100644 --- a/backend/app/routers/public.py +++ b/backend/app/routers/public.py @@ -63,10 +63,12 @@ ) from app.services import ai_bridge from app.services.extraction import ( + ExtractionError, extract_from_letter_file, generate_reply_text, normalize_lang, ) +from app.services.pdf_pages import PdfRenderError from app.services.persistence import persist_extraction from app.services.storage import detect_magic_mime, save_letter_file @@ -257,6 +259,24 @@ async def post_letter( extracted = await extract_from_letter_file( saved_path, actual_mime, lang=out_lang ) + except PdfRenderError as exc: + # PDF couldn't be rendered to page images (corrupt / password-protected + # / poppler missing). Surface the distinct, actionable message + # ("try uploading it as an image instead") instead of a generic 500. + logger.info("PDF render failed for letter %s: %s", letter.id, exc) + letter.status = LetterStatus.ERROR + db.add(letter) + db.commit() + raise KlarHTTPException(502, ErrorCode.PDF_RENDER_FAILED, message=str(exc)) + except ExtractionError as exc: + # Scanned, image-only PDF with no readable text layer, or malformed + # model output — the issue-#8 crash. Return the typed, user-friendly + # message rather than letting a raw exception become a 500. + logger.info("Extraction produced no text for letter %s: %s", letter.id, exc) + letter.status = LetterStatus.ERROR + db.add(letter) + db.commit() + raise KlarHTTPException(502, ErrorCode.EXTRACTION_FAILED, message=str(exc)) except Exception as exc: # Log the real Qwen error to the server console so we can diagnose. # The 502 response stays generic on the wire to avoid leaking provider diff --git a/backend/app/services/extraction.py b/backend/app/services/extraction.py index dc9c377..1c04fc6 100644 --- a/backend/app/services/extraction.py +++ b/backend/app/services/extraction.py @@ -28,6 +28,15 @@ DOCUMENT_CATEGORIES: list[str] = [c.value for c in DocumentCategory] + +class ExtractionError(Exception): + """Raised when the vision model can't produce usable structured output. + + Most common cause: a scanned, image-only PDF with no readable text layer, + where the model returns no tool call. Callers map this to a friendly + `EXTRACTION_FAILED` error instead of letting a raw exception become a 500. + """ + # ISO 639-1 codes — matches the frontend's docs/06-frontend-integration-contract.md. # Qwen3.7-Plus handles all of these out of the box. Quality bar: # en/de: production-grade, the wedge languages @@ -226,6 +235,13 @@ async def extract_from_letter_file( sent as a separate image_url content part so the model sees the whole doc. """ pages = split_to_image_bytes(path, mime) + # Empty-page guard: a PDF that renders to zero pages (or an empty file) + # would otherwise send the model an image-less prompt and waste a call. + if not pages: + raise ExtractionError( + "Could not read this document — it produced no pages. It may be a " + "blank or unreadable scan." + ) image_parts = list(iter_data_urls(pages)) rag_hits = store.search( @@ -267,10 +283,20 @@ async def extract_from_letter_file( tool_calls = response.choices[0].message.tool_calls or [] if not tool_calls: - raise RuntimeError( - "Model returned no tool call; check model + prompt compatibility." + # The model couldn't find structured content to extract. For a scanned, + # image-only PDF with no readable text this is the expected outcome — + # surface it as a typed error the caller turns into a friendly message + # rather than a raw 500. + raise ExtractionError( + "Could not extract text from this document. It may be a scanned " + "image without readable content — try a clearer photo or PDF." ) - payload = json.loads(tool_calls[0].function.arguments) + try: + payload = json.loads(tool_calls[0].function.arguments) + except (json.JSONDecodeError, TypeError) as exc: + raise ExtractionError( + "The document reader returned a malformed response. Please try again." + ) from exc # Defensive: models can emit `actions` as # - a proper list of dicts (happy path) diff --git a/backend/app/services/pdf_pages.py b/backend/app/services/pdf_pages.py index e0698e6..8f4ae61 100644 --- a/backend/app/services/pdf_pages.py +++ b/backend/app/services/pdf_pages.py @@ -9,20 +9,47 @@ from typing import Iterable +class PdfRenderError(Exception): + """Raised when a PDF cannot be rendered to page images. + + Covers a missing/broken poppler install (`pdf2image` raises + `PDFInfoNotInstalledError`), a corrupt or password-protected file + (`PDFPageCountError` / `PDFSyntaxError`), and the edge case where poppler + succeeds but produces zero pages. Callers catch this to surface a + user-friendly `PDF_RENDER_FAILED` error instead of a raw 500. + """ + + def pdf_to_image_bytes(path: str, *, dpi: int = 200, max_pages: int = 12) -> list[bytes]: """Render up to `max_pages` pages of `path` to PNG bytes. Imported lazily so callers that never touch a PDF don't pay the pdf2image / poppler import cost. + + Raises `PdfRenderError` if the PDF can't be rendered (corrupt file, + poppler missing) or renders to zero pages — NEVER lets a raw pdf2image + exception escape, so upstream callers can map it to a friendly error. """ from pdf2image import convert_from_path - pages = convert_from_path(path, dpi=dpi, first_page=1, last_page=max_pages) + try: + pages = convert_from_path(path, dpi=dpi, first_page=1, last_page=max_pages) + except Exception as exc: # noqa: BLE001 — normalize every pdf2image failure + raise PdfRenderError( + "Could not render this PDF. It may be corrupt, password-protected, " + "or the server is missing poppler." + ) from exc + out: list[bytes] = [] for img in pages: buf = BytesIO() img.save(buf, format="PNG") out.append(buf.getvalue()) + + if not out: + # poppler returned no pages — an empty or unreadable document. + raise PdfRenderError("This PDF has no readable pages.") + return out diff --git a/backend/pytest.ini b/backend/pytest.ini new file mode 100644 index 0000000..78c5011 --- /dev/null +++ b/backend/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +asyncio_mode = auto +testpaths = tests diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..994bc92 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,37 @@ +"""Shared pytest setup for backend tests. + +Sets up an isolated SQLite DB and makes the repo-root `ai/` package importable +(the production code does the same in `app.main._ensure_ai_package_importable`) +BEFORE any `app.*` module is imported, so module-level `create_engine` and +`from ai... import` calls resolve against the test environment. +""" + +import os +import sys +import tempfile +from pathlib import Path + +# --- make the repo root importable so `import ai...` works in tests --- +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +# --- isolate the DB to a throwaway file before app.database is imported --- +_TMP_DB = Path(tempfile.gettempdir()) / "klar_test.db" +if _TMP_DB.exists(): + _TMP_DB.unlink() +os.environ.setdefault("DATABASE_URL", f"sqlite:///{_TMP_DB}") +os.environ.setdefault("JWT_SECRET", "test-secret-test-secret-test-secret-32") + +import pytest # noqa: E402 + +from app.database import engine, init_db # noqa: E402 + + +@pytest.fixture(scope="session", autouse=True) +def _create_schema(): + init_db() + yield + engine.dispose() + if _TMP_DB.exists(): + _TMP_DB.unlink() diff --git a/backend/tests/test_scanned_pdf_graceful.py b/backend/tests/test_scanned_pdf_graceful.py new file mode 100644 index 0000000..4a07122 --- /dev/null +++ b/backend/tests/test_scanned_pdf_graceful.py @@ -0,0 +1,356 @@ +"""Regression tests for issue #8 — scanned PDFs with no text layer must NOT 500. + +The reported crash happened when uploading a scanned, image-only PDF: the +extraction step raised a raw exception that bubbled up as a 500 instead of a +user-friendly error. These tests pin every extraction entry point to a typed, +graceful failure. +""" + +import sys +import types +from uuid import uuid4 + +import pytest + +from ai.react_agent.ocr import OcrError, _parse_ocr_response +from app.database import engine +from app.errors import ErrorCode +from app.models import Letter, LetterStatus +from app.pipeline import orchestrator +from app.services import extraction +from app.services.extraction import ExtractionError +from app.services.pdf_pages import PdfRenderError, pdf_to_image_bytes +from sqlmodel import Session + + +# -------------------------------------------------------------------------- +# 1. ai/react_agent/ocr.py — malformed / blank OCR responses → OcrError +# -------------------------------------------------------------------------- + + +def test_parse_ocr_response_happy_path(): + result = {"choices": [{"message": {"content": "Sehr geehrte Damen und Herren"}}]} + assert _parse_ocr_response(result) == "Sehr geehrte Damen und Herren" + + +@pytest.mark.parametrize( + "result", + [ + "not a dict", + {}, + {"choices": []}, + {"choices": [{}]}, + {"choices": [{"message": None}]}, + {"choices": [{"message": {"content": None}}]}, + {"choices": [{"message": {"content": " "}}]}, + ], +) +def test_parse_ocr_response_malformed_raises_ocrerror(result): + with pytest.raises(OcrError): + _parse_ocr_response(result) + + +# -------------------------------------------------------------------------- +# 2. pdf_pages.pdf_to_image_bytes — render failures → PdfRenderError +# -------------------------------------------------------------------------- + + +def test_pdf_to_image_bytes_render_exception(monkeypatch): + import pdf2image + + def _boom(*a, **k): + raise RuntimeError("poppler exploded") + + monkeypatch.setattr(pdf2image, "convert_from_path", _boom) + with pytest.raises(PdfRenderError): + pdf_to_image_bytes("/tmp/whatever.pdf") + + +def test_pdf_to_image_bytes_zero_pages(monkeypatch): + import pdf2image + + monkeypatch.setattr(pdf2image, "convert_from_path", lambda *a, **k: []) + with pytest.raises(PdfRenderError): + pdf_to_image_bytes("/tmp/empty.pdf") + + +# -------------------------------------------------------------------------- +# 3. extraction.extract_from_letter_file — no tool call / empty pages +# -------------------------------------------------------------------------- + + +class _FakeMessage: + def __init__(self, tool_calls): + self.tool_calls = tool_calls + + +class _FakeChoice: + def __init__(self, tool_calls): + self.message = _FakeMessage(tool_calls) + + +class _FakeResponse: + def __init__(self, tool_calls): + self.choices = [_FakeChoice(tool_calls)] + + +class _FakeCompletions: + def __init__(self, tool_calls): + self._tool_calls = tool_calls + + async def create(self, **kwargs): + return _FakeResponse(self._tool_calls) + + +class _FakeClient: + def __init__(self, tool_calls): + self.chat = types.SimpleNamespace(completions=_FakeCompletions(tool_calls)) + + +async def test_extract_from_letter_file_no_tool_call(monkeypatch): + monkeypatch.setattr( + extraction, "split_to_image_bytes", lambda p, m: [(b"\x89PNG", "image/png")] + ) + monkeypatch.setattr(extraction.store, "search", lambda *a, **k: []) + monkeypatch.setattr(extraction, "_get_client", lambda: _FakeClient(tool_calls=[])) + + with pytest.raises(ExtractionError): + await extraction.extract_from_letter_file("/tmp/scan.pdf", "application/pdf") + + +async def test_extract_from_letter_file_empty_pages(monkeypatch): + monkeypatch.setattr(extraction, "split_to_image_bytes", lambda p, m: []) + with pytest.raises(ExtractionError): + await extraction.extract_from_letter_file("/tmp/scan.pdf", "application/pdf") + + +# -------------------------------------------------------------------------- +# 4. orchestrator._ocr_letter_file — handles PDFs + blank scans gracefully +# -------------------------------------------------------------------------- + + +async def test_ocr_letter_file_image_blank(monkeypatch): + async def _blank(path): + return " " + + monkeypatch.setattr("ai.react_agent.ocr.extract_text_from_image", _blank) + with pytest.raises(ExtractionError): + await orchestrator._ocr_letter_file("/tmp/photo.jpg") + + +async def test_ocr_letter_file_pdf_all_pages_blank(monkeypatch): + monkeypatch.setattr( + orchestrator, "pdf_to_image_bytes", lambda p, **k: [b"png1", b"png2"] + ) + + async def _blank(path): + return "" + + monkeypatch.setattr("ai.react_agent.ocr.extract_text_from_image", _blank) + with pytest.raises(ExtractionError): + await orchestrator._ocr_letter_file("/tmp/scan.pdf") + + +async def test_ocr_letter_file_pdf_render_error_propagates(monkeypatch): + def _boom(p, **k): + raise PdfRenderError("corrupt") + + monkeypatch.setattr(orchestrator, "pdf_to_image_bytes", _boom) + with pytest.raises(PdfRenderError): + await orchestrator._ocr_letter_file("/tmp/scan.pdf") + + +async def test_ocr_letter_file_pdf_joins_pages(monkeypatch): + monkeypatch.setattr( + orchestrator, "pdf_to_image_bytes", lambda p, **k: [b"png1", b"png2"] + ) + + async def _ocr(path): + return "page text" + + monkeypatch.setattr("ai.react_agent.ocr.extract_text_from_image", _ocr) + text = await orchestrator._ocr_letter_file("/tmp/scan.pdf") + assert text.count("page text") == 2 + + +# -------------------------------------------------------------------------- +# 5. Full SSE pipeline regression — scanned PDF yields a graceful `error` +# event (NOT an unhandled 500) and marks the letter ERROR. +# -------------------------------------------------------------------------- + + +def _install_fake_agent_module(): + """The orchestrator imports `ai.react_agent.agent` (which needs API keys + to instantiate) before OCR runs. Inject a stub so the import guard passes + without real credentials.""" + mod = types.ModuleType("ai.react_agent.agent") + + async def run_react_agent(ocr_text): # pragma: no cover - never reached here + if False: + yield None + + mod.run_react_agent = run_react_agent + sys.modules["ai.react_agent.agent"] = mod + + +async def test_process_letter_stream_scanned_pdf_emits_error(monkeypatch): + _install_fake_agent_module() + + # Persist a letter pointing at a (notional) scanned PDF. + with Session(engine) as db: + letter = Letter( + user_id=uuid4(), + language="en", + status=LetterStatus.UPLOADED, + original_file="/tmp/scanned-no-text-layer.pdf", + ) + db.add(letter) + db.commit() + db.refresh(letter) + letter_id = letter.id + + # OCR can't read the scan → typed ExtractionError. + async def _no_text(path): + raise ExtractionError("scanned image without readable content") + + monkeypatch.setattr(orchestrator, "_ocr_letter_file", _no_text) + + events = [ + chunk async for chunk in orchestrator.process_letter_stream(letter_id, "en") + ] + blob = "".join(events) + + assert "event: error" in blob + assert ErrorCode.EXTRACTION_FAILED.value in blob + # The user-facing message is surfaced, not a stack trace / 500. + assert "readable content" in blob + + with Session(engine) as db: + refreshed = db.get(Letter, letter_id) + assert refreshed.status == LetterStatus.ERROR + + +# -------------------------------------------------------------------------- +# 6. Production POST /letters (public.py) — the exact `/api/letters` endpoint +# named in the issue. A scanned/corrupt PDF must yield a typed 502 with the +# right ErrorCode + user-facing message, NOT a raw 500. +# -------------------------------------------------------------------------- + + +class _FakeUploadFile: + """Minimal stand-in for fastapi.UploadFile for the upload handler.""" + + def __init__(self, data: bytes, content_type: str): + self._data = data + self.content_type = content_type + + async def read(self) -> bytes: + return self._data + + +def _make_user(db): + from app.models import User + + user = User(email=f"scan-{uuid4()}@example.com", language="en") + db.add(user) + db.commit() + db.refresh(user) + return user + + +async def _call_post_letter(monkeypatch, *, raise_exc): + """Drive public.post_letter with extraction stubbed to raise `raise_exc`. + + Returns the KlarHTTPException the handler raises (or None if it didn't). + """ + from app.routers import public + + # A real PDF magic-number so the upload validation passes without poppler. + monkeypatch.setattr(public, "detect_magic_mime", lambda data: "application/pdf") + monkeypatch.setattr( + public, "save_letter_file", lambda *a, **k: "/tmp/scanned-no-text.pdf" + ) + + async def _boom(*a, **k): + raise raise_exc + + monkeypatch.setattr(public, "extract_from_letter_file", _boom) + + upload = _FakeUploadFile(b"%PDF-1.4 fake bytes", "application/pdf") + + with Session(engine) as db: + user = _make_user(db) + try: + await public.post_letter(file=upload, lang="en", db=db, user=user) + except Exception as exc: # noqa: BLE001 — we assert on the typed error + return exc + return None + + +async def test_post_letter_scanned_pdf_returns_extraction_failed(monkeypatch): + from app.errors import KlarHTTPException + + exc = await _call_post_letter( + monkeypatch, + raise_exc=ExtractionError("scanned image without readable content"), + ) + assert isinstance(exc, KlarHTTPException) + assert exc.status_code == 502 + assert exc.code == ErrorCode.EXTRACTION_FAILED + assert "readable content" in exc.message + + +async def test_post_letter_corrupt_pdf_returns_pdf_render_failed(monkeypatch): + from app.errors import KlarHTTPException + + exc = await _call_post_letter( + monkeypatch, + raise_exc=PdfRenderError("Could not render this PDF. It may be corrupt."), + ) + assert isinstance(exc, KlarHTTPException) + assert exc.status_code == 502 + # Distinct, actionable code (not lumped into the generic EXTRACTION_FAILED). + assert exc.code == ErrorCode.PDF_RENDER_FAILED + assert "corrupt" in exc.message + + +async def test_post_letter_unexpected_error_stays_generic(monkeypatch): + from app.errors import KlarHTTPException + + exc = await _call_post_letter( + monkeypatch, raise_exc=RuntimeError("some provider 500 with secrets") + ) + assert isinstance(exc, KlarHTTPException) + assert exc.status_code == 502 + assert exc.code == ErrorCode.EXTRACTION_FAILED + # Raw provider error must NOT leak into the user-facing message. + assert "secrets" not in exc.message + + +async def test_process_letter_stream_pdf_render_failure_emits_error(monkeypatch): + _install_fake_agent_module() + + with Session(engine) as db: + letter = Letter( + user_id=uuid4(), + language="en", + status=LetterStatus.UPLOADED, + original_file="/tmp/corrupt.pdf", + ) + db.add(letter) + db.commit() + db.refresh(letter) + letter_id = letter.id + + async def _render_fail(path): + raise PdfRenderError("could not render this PDF") + + monkeypatch.setattr(orchestrator, "_ocr_letter_file", _render_fail) + + events = [ + chunk async for chunk in orchestrator.process_letter_stream(letter_id, "en") + ] + blob = "".join(events) + + assert "event: error" in blob + assert ErrorCode.PDF_RENDER_FAILED.value in blob