diff --git a/Dockerfile b/Dockerfile index d0bd838..e435af6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update -y && apt-get upgrade -y \ gcc \ libpq-dev \ libxml2-dev \ - python3-dev \ + python3-dev \ postgresql-client \ && rm -rf /var/lib/apt/lists/ @@ -45,3 +45,20 @@ RUN pip install --no-cache-dir -e . EXPOSE 8000 CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0", "quiabo:app"] + +FROM app AS worker + +USER root + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends \ + tesseract-ocr \ + tesseract-ocr-eng \ + tesseract-ocr-spa \ + tesseract-ocr-deu \ + tesseract-ocr-chi-tra \ + tesseract-ocr-ita \ + tesseract-ocr-fra \ + && rm -rf /var/lib/apt/lists/* + +USER $APP_USER diff --git a/docker-compose.yml b/docker-compose.yml index 1d40c3f..63e9f59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,6 @@ x-quiabo-common: QUIABO_CELERY__task_default_queue: ${QUIABO_CELERY__task_default_queue:-quiabo} QUIABO_CELERY__task_ignore_result: ${QUIABO_CELERY__task_ignore_result:-false} - services: db: environment: @@ -39,7 +38,7 @@ services: redis: condition: service_healthy environment: - <<: + <<: - *quiabo-common-environment - *dbconfig init: true @@ -48,10 +47,12 @@ services: - 8000:8000 volumes: - ./quiabo:/app/quiabo:rw + - ./test:/app/test:rw worker: build: context: . + target: worker depends_on: db: condition: service_healthy @@ -66,6 +67,7 @@ services: command: celery -A quiabo.celery_app worker --loglevel INFO volumes: - ./quiabo:/app/quiabo:rw + - ./files:/app/files:rw flower: build: diff --git a/files/.keep b/files/.keep new file mode 100644 index 0000000..e69de29 diff --git a/quiabo/celery.py b/quiabo/celery.py index 6d016b3..d0837a9 100644 --- a/quiabo/celery.py +++ b/quiabo/celery.py @@ -32,5 +32,6 @@ def __call__(self, *args: object, **kwargs: object) -> object: celery_app = Celery(app.name, task_cls=FlaskTask) celery_app.config_from_object(app.config["CELERY"]) celery_app.set_default() + celery_app.autodiscover_tasks(["quiabo"], force=True) app.extensions["celery"] = celery_app return celery_app diff --git a/quiabo/tasks.py b/quiabo/tasks.py new file mode 100644 index 0000000..348586e --- /dev/null +++ b/quiabo/tasks.py @@ -0,0 +1,28 @@ +"""Celery tasks for running OCR jobs.""" + +import subprocess +from pathlib import Path + +from celery import shared_task + + +@shared_task(bind=True) +def run_tesseract_job(self, filelist: str, languages: list[str], output: str) -> dict: + """Run Tesseract over the files listed in filelist and write output PDF.""" + + output = output.removesuffix(".pdf") + + output_dir = Path(output).parent + if not output_dir.exists(): + raise FileNotFoundError(f"Output directory {output_dir} does not exist.") + + # The state update does not need any meta, unless we want it for debugging or tracking. + self.update_state( + state="STARTED", + meta={"filelist": filelist, "languages": languages, "output": output}, + ) + + command = ["tesseract", "-l", "+".join(languages), filelist, output, "pdf"] + subprocess.run(command, check=True, capture_output=True, text=True) + + return {"output": f"{output}.pdf"} diff --git a/test/fixtures/files/files.txt b/test/fixtures/files/files.txt new file mode 100644 index 0000000..0e2a2d3 --- /dev/null +++ b/test/fixtures/files/files.txt @@ -0,0 +1 @@ +/app/test/fixture/files/testocr.png diff --git a/test/fixtures/files/testocr.png b/test/fixtures/files/testocr.png new file mode 100644 index 0000000..ce8d0e7 Binary files /dev/null and b/test/fixtures/files/testocr.png differ diff --git a/test/unit/test_tasks.py b/test/unit/test_tasks.py new file mode 100644 index 0000000..3673846 --- /dev/null +++ b/test/unit/test_tasks.py @@ -0,0 +1,84 @@ +"""Unit tests for the shared task defined in ``tasks.py``.""" + +from unittest.mock import Mock + +from quiabo.tasks import run_tesseract_job + + +def test_shared_task_is_registered(): + """Verify that the Tesseract task is registered with Celery.""" + task = run_tesseract_job + + assert task.name + assert task.app.tasks[task.name].name == task.name + assert callable(task.run) + + +def test_shared_task_delay_delegates_to_apply_async(monkeypatch): + """Verify that delaying the task delegates to ``apply_async``.""" + apply_async = Mock(return_value="queued") + monkeypatch.setattr(run_tesseract_job, "apply_async", apply_async) + + result = run_tesseract_job.delay("filelist.txt", ["eng", "spa"], "output") + + assert result == "queued" + apply_async.assert_called_once_with( + ("filelist.txt", ["eng", "spa"], "output"), + {}, + ) + + +def test_run_tesseract_job_updates_state_and_runs_tesseract(monkeypatch): + """Verify that the task updates state and invokes Tesseract.""" + update_state = Mock() + run = Mock() + monkeypatch.setattr(run_tesseract_job, "update_state", update_state) + monkeypatch.setattr("quiabo.tasks.subprocess.run", run) + + result = run_tesseract_job.run( + "files.txt", + ["eng", "spa"], + "output", + ) + + update_state.assert_called_once_with( + state="STARTED", + meta={ + "filelist": "files.txt", + "languages": ["eng", "spa"], + "output": "output", + }, + ) + run.assert_called_once_with( + ["tesseract", "-l", "eng+spa", "files.txt", "output", "pdf"], + check=True, + capture_output=True, + text=True, + ) + assert result == {"output": "output.pdf"} + + +def test_run_tesseract_job_removes_pdf_suffix(monkeypatch): + """Verify that an existing PDF suffix is removed before invocation.""" + update_state = Mock() + run = Mock() + monkeypatch.setattr(run_tesseract_job, "update_state", update_state) + monkeypatch.setattr("quiabo.tasks.subprocess.run", run) + + result = run_tesseract_job.run("files.txt", ["eng"], "output.pdf") + + update_state.assert_called_once_with( + state="STARTED", + meta={ + "filelist": "files.txt", + "languages": ["eng"], + "output": "output", + }, + ) + run.assert_called_once_with( + ["tesseract", "-l", "eng", "files.txt", "output", "pdf"], + check=True, + capture_output=True, + text=True, + ) + assert result == {"output": "output.pdf"}