From 8fd950bd12fe9f627efcbadb634e9a78f850b777 Mon Sep 17 00:00:00 2001 From: Sanjin <102841251+duckduckgrayduck@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:36:31 -0600 Subject: [PATCH 1/3] Skip grafting if DISABLE_GRAFTING is set --- config/settings/base.py | 1 + documentcloud/documents/models/document.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index cfdc6e47..c74364ad 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -717,6 +717,7 @@ # ------------------------------------------------------------------------------ MAX_PAGES = env.int("MAX_PAGES", default=50) GRAFT_DEBUG = env.bool("GRAFT_DEBUG", default=False) +DISABLE_GRAFTING = env.bool("DISABLE_GRAFTING", default=False) # STATS API UPLOAD_WINDOW_DAYS = env.int("UPLOAD_WINDOW_DAYS", default=90) diff --git a/documentcloud/documents/models/document.py b/documentcloud/documents/models/document.py index d773a07c..aa68fc4a 100644 --- a/documentcloud/documents/models/document.py +++ b/documentcloud/documents/models/document.py @@ -503,14 +503,20 @@ def set_page_text(self, page_text_infos): def _set_page_positions(self, pages, file_names, file_contents): """Handle grafting page positions back into the document""" - current_pdf = pymupdf.open(stream=storage.open(self.doc_path, "rb").read()) - start_page = pages[0]["page_number"] - stop_page = pages[-1]["page_number"] # always write the position JSON - this is cheap and does not cause the # memory issues that gating below guards against self._write_position_json(pages, file_names, file_contents) + # If this flag is set, skip grafting entirely. + if settings.DISABLE_GRAFTING: + logger.info("[SET PAGE TEXT] %d - grafting disabled, skipping", self.pk) + return None + + current_pdf = pymupdf.open(stream=storage.open(self.doc_path, "rb").read()) + start_page = pages[0]["page_number"] + stop_page = pages[-1]["page_number"] + visible_text = self._check_visible_text(current_pdf, start_page, stop_page) logger.info( "[SET PAGE TEXT] %d - visible text detected: %s", self.pk, visible_text From 058be118b0e5fe3f3985f9afd3e88f4df9cbe550 Mon Sep 17 00:00:00 2001 From: Sanjin <102841251+duckduckgrayduck@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:15:35 -0600 Subject: [PATCH 2/3] Add DISABLE_TEXTRACT_GRAFTING lambda parameter and skip grafting pages with visible text like the Django path does --- config/aws/lambda/template_params.yaml | 1 + .../processing/info_and_image/main.py | 44 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/config/aws/lambda/template_params.yaml b/config/aws/lambda/template_params.yaml index 8c6041d3..5a5dc117 100644 --- a/config/aws/lambda/template_params.yaml +++ b/config/aws/lambda/template_params.yaml @@ -39,6 +39,7 @@ Globals: RETRY_ERROR_TOPIC: "{{resolve:ssm:/dc/{$ENV$}/lambdas/retry_errors/topic:latest}}" PDF_SIZE_LIMIT: "{{resolve:ssm:/dc/{$ENV$}/params/pdf_size_limit:latest}}" PAGE_COUNT_LIMIT: "{{resolve:ssm:/dc/{$ENV$}/params/page_count_limit:latest}}" + DISABLE_TEXTRACT_GRAFTING: "{{resolve:ssm:/dc/{$ENV$}/params/disable_textract_grafting:latest}}" BLOCK_SIZE: "{{resolve:ssm:/dc/{$ENV$}/params/block_size:latest}}" IMPORT_DIR: "{{resolve:ssm:/dc/{$ENV$}/lambdas/config/import_dir:latest}}" IMPORT_BUCKET: "{{resolve:ssm:/dc/{$ENV$}/lambdas/config/import_bucket:latest}}" diff --git a/documentcloud/documents/processing/info_and_image/main.py b/documentcloud/documents/processing/info_and_image/main.py index 3e9afd1e..d3b7eb67 100755 --- a/documentcloud/documents/processing/info_and_image/main.py +++ b/documentcloud/documents/processing/info_and_image/main.py @@ -96,6 +96,8 @@ PDF_SIZE_LIMIT = env.int("PDF_SIZE_LIMIT", 501 * 1024 * 1024) # PDF page count limit, default 2000 PAGE_COUNT_LIMIT = env.int("PAGE_COUNT_LIMIT", 2000) +# Disable grafting for textract, for debugging +DISABLE_TEXTRACT_GRAFTING = env.bool("DISABLE_TEXTRACT_GRAFTING", default=False) BLOCK_SIZE = env.int( "BLOCK_SIZE", 8 * 1024 * 1024 @@ -553,37 +555,67 @@ def graft_ocr_in_pdf_tess(doc_id, slug, access): grafter.finalize() +def has_visible_text(page): + """Check if a pymupdf page contains visible text""" + for trace in page.get_texttrace(): + # zero opacity is transparent text and type 3 is hidden text + if trace["opacity"] != 0 and trace["type"] != 3: + return True + return False + + def graft_ocr_in_pdf(doc_id, slug, access): """Reinjects the OCR'd text-only PDFs back into the main PDF. This uses overlay PDFs generated by PyMuPdf using the position information. It is suitable for non-tesseract OCR. """ + + if DISABLE_TEXTRACT_GRAFTING: + logger.info("[GRAFT] doc_id %s textract grafting disabled, skipping", doc_id) + return + logger.info("[GRAFT] doc_id %s textract grafting enabled", doc_id) + page_text_pdf_field = redis_fields.page_text_pdf(doc_id) redis_pdf_pages = REDIS.hkeys(page_text_pdf_field) doc_path = path.doc_path(doc_id, slug) base_pdf = pymupdf.open(stream=storage.open(doc_path).read()) - buffer = io.BytesIO() + graft_pages = [] for redis_page_key in redis_pdf_pages: page_number = int(redis_page_key) page = base_pdf[page_number] + if has_visible_text(page): + # redacting would delete the page's real text + logger.info( + "[GRAFT] doc_id %s page %s has visible text, skipping", + doc_id, + page_number, + ) + continue page.add_redact_annot(page.rect) page.apply_redactions( images=pymupdf.PDF_REDACT_IMAGE_NONE, graphics=pymupdf.PDF_REDACT_LINE_ART_NONE, ) + graft_pages.append((redis_page_key, page_number)) + + if not graft_pages: + logger.info("[GRAFT] doc_id %s no pages to graft", doc_id) + base_pdf.close() + return + + buffer = io.BytesIO() base_pdf.save(buffer) + base_pdf.close() buffer.seek(0) base_pdf = Pdf.open(buffer) - for redis_page_key in redis_pdf_pages: - page_number = int(redis_page_key) + for redis_page_key, page_number in graft_pages: overlay_pdf = Pdf.open( io.BytesIO(REDIS.hget(page_text_pdf_field, redis_page_key)) ) - base_pdf.pages[page_number].add_overlay( - overlay_pdf.pages[0], Rectangle(*base_pdf.pages[0].trimbox) - ) + base_page = base_pdf.pages[page_number] + base_page.add_overlay(overlay_pdf.pages[0], Rectangle(*base_page.trimbox)) # Overwrite source PDF mem_file = io.BytesIO() From 30c0d5849fe14095d4ccc62858d33da922684b1b Mon Sep 17 00:00:00 2001 From: Sanjin <102841251+duckduckgrayduck@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:20:05 -0600 Subject: [PATCH 3/3] Disable pylint check --- documentcloud/documents/processing/info_and_image/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentcloud/documents/processing/info_and_image/main.py b/documentcloud/documents/processing/info_and_image/main.py index d3b7eb67..2d189aa7 100755 --- a/documentcloud/documents/processing/info_and_image/main.py +++ b/documentcloud/documents/processing/info_and_image/main.py @@ -569,7 +569,7 @@ def graft_ocr_in_pdf(doc_id, slug, access): This uses overlay PDFs generated by PyMuPdf using the position information. It is suitable for non-tesseract OCR. """ - + # pylint:disable=too-many-locals if DISABLE_TEXTRACT_GRAFTING: logger.info("[GRAFT] doc_id %s textract grafting disabled, skipping", doc_id) return