Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions config/aws/lambda/template_params.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ Globals:
RETRY_ERROR_TOPIC: "{{resolve:ssm:/dc/{$ENV$}/lambdas/retry_errors/topic:latest}}"
PDF_SIZE_LIMIT: "{{resolve:ssm:/dc/{$ENV$}/params/pdf_size_limit:latest}}"
PAGE_COUNT_LIMIT: "{{resolve:ssm:/dc/{$ENV$}/params/page_count_limit:latest}}"
DISABLE_TEXTRACT_GRAFTING: "{{resolve:ssm:/dc/{$ENV$}/params/disable_textract_grafting:latest}}"
BLOCK_SIZE: "{{resolve:ssm:/dc/{$ENV$}/params/block_size:latest}}"
IMPORT_DIR: "{{resolve:ssm:/dc/{$ENV$}/lambdas/config/import_dir:latest}}"
IMPORT_BUCKET: "{{resolve:ssm:/dc/{$ENV$}/lambdas/config/import_bucket:latest}}"
Expand Down
1 change: 1 addition & 0 deletions config/settings/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,7 @@
# ------------------------------------------------------------------------------
MAX_PAGES = env.int("MAX_PAGES", default=50)
GRAFT_DEBUG = env.bool("GRAFT_DEBUG", default=False)
DISABLE_GRAFTING = env.bool("DISABLE_GRAFTING", default=False)

# STATS API
UPLOAD_WINDOW_DAYS = env.int("UPLOAD_WINDOW_DAYS", default=90)
Expand Down
12 changes: 9 additions & 3 deletions documentcloud/documents/models/document.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,14 +503,20 @@ def set_page_text(self, page_text_infos):

def _set_page_positions(self, pages, file_names, file_contents):
"""Handle grafting page positions back into the document"""
current_pdf = pymupdf.open(stream=storage.open(self.doc_path, "rb").read())
start_page = pages[0]["page_number"]
stop_page = pages[-1]["page_number"]

# always write the position JSON - this is cheap and does not cause the
# memory issues that gating below guards against
self._write_position_json(pages, file_names, file_contents)

# If this flag is set, skip grafting entirely.
if settings.DISABLE_GRAFTING:
logger.info("[SET PAGE TEXT] %d - grafting disabled, skipping", self.pk)
return None

current_pdf = pymupdf.open(stream=storage.open(self.doc_path, "rb").read())
start_page = pages[0]["page_number"]
stop_page = pages[-1]["page_number"]

visible_text = self._check_visible_text(current_pdf, start_page, stop_page)
logger.info(
"[SET PAGE TEXT] %d - visible text detected: %s", self.pk, visible_text
Expand Down
44 changes: 38 additions & 6 deletions documentcloud/documents/processing/info_and_image/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@
PDF_SIZE_LIMIT = env.int("PDF_SIZE_LIMIT", 501 * 1024 * 1024)
# PDF page count limit, default 2000
PAGE_COUNT_LIMIT = env.int("PAGE_COUNT_LIMIT", 2000)
# Disable grafting for textract, for debugging
DISABLE_TEXTRACT_GRAFTING = env.bool("DISABLE_TEXTRACT_GRAFTING", default=False)

BLOCK_SIZE = env.int(
"BLOCK_SIZE", 8 * 1024 * 1024
Expand Down Expand Up @@ -553,37 +555,67 @@ def graft_ocr_in_pdf_tess(doc_id, slug, access):
grafter.finalize()


def has_visible_text(page):
"""Check if a pymupdf page contains visible text"""
for trace in page.get_texttrace():
# zero opacity is transparent text and type 3 is hidden text
if trace["opacity"] != 0 and trace["type"] != 3:
return True
return False


def graft_ocr_in_pdf(doc_id, slug, access):
"""Reinjects the OCR'd text-only PDFs back into the main PDF.
This uses overlay PDFs generated by PyMuPdf using the position information.
It is suitable for non-tesseract OCR.
"""
# pylint:disable=too-many-locals
if DISABLE_TEXTRACT_GRAFTING:
logger.info("[GRAFT] doc_id %s textract grafting disabled, skipping", doc_id)
return
logger.info("[GRAFT] doc_id %s textract grafting enabled", doc_id)

page_text_pdf_field = redis_fields.page_text_pdf(doc_id)
redis_pdf_pages = REDIS.hkeys(page_text_pdf_field)
doc_path = path.doc_path(doc_id, slug)

base_pdf = pymupdf.open(stream=storage.open(doc_path).read())
buffer = io.BytesIO()
graft_pages = []
for redis_page_key in redis_pdf_pages:
page_number = int(redis_page_key)
page = base_pdf[page_number]
if has_visible_text(page):
# redacting would delete the page's real text
logger.info(
"[GRAFT] doc_id %s page %s has visible text, skipping",
doc_id,
page_number,
)
continue
page.add_redact_annot(page.rect)
page.apply_redactions(
images=pymupdf.PDF_REDACT_IMAGE_NONE,
graphics=pymupdf.PDF_REDACT_LINE_ART_NONE,
)
graft_pages.append((redis_page_key, page_number))

if not graft_pages:
logger.info("[GRAFT] doc_id %s no pages to graft", doc_id)
base_pdf.close()
return

buffer = io.BytesIO()
base_pdf.save(buffer)
base_pdf.close()
buffer.seek(0)

base_pdf = Pdf.open(buffer)
for redis_page_key in redis_pdf_pages:
page_number = int(redis_page_key)
for redis_page_key, page_number in graft_pages:
overlay_pdf = Pdf.open(
io.BytesIO(REDIS.hget(page_text_pdf_field, redis_page_key))
)
base_pdf.pages[page_number].add_overlay(
overlay_pdf.pages[0], Rectangle(*base_pdf.pages[0].trimbox)
)
base_page = base_pdf.pages[page_number]
base_page.add_overlay(overlay_pdf.pages[0], Rectangle(*base_page.trimbox))

# Overwrite source PDF
mem_file = io.BytesIO()
Expand Down
Loading