diff --git a/CHANGELOG.md b/CHANGELOG.md index af01c09..f941fcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,11 @@ This changelog records user-visible changes. Vectorless RAG follows semantic ver ## Unreleased -No changes have been recorded after v0.3.0. +### Fixed + +- Replaced unmaintained `PyPDF2` with maintained `pypdf` across the API and vendored PageIndex runtime +- Updated the vulnerable transitive `brace-expansion` package to its quarantined patched release +- Made the PageIndex installer resolve its patch path correctly when invoked through a relative path ## 0.3.0, 2026-07-31 diff --git a/apps/api/patches/pageindex-runtime.patch b/apps/api/patches/pageindex-runtime.patch index cb035cb..e1d1a38 100644 --- a/apps/api/patches/pageindex-runtime.patch +++ b/apps/api/patches/pageindex-runtime.patch @@ -1,3 +1,25 @@ +diff --git a/pageindex/client.py b/pageindex/client.py +index 894dab1..ec5ec3b 100644 +--- a/pageindex/client.py ++++ b/pageindex/client.py +@@ -5,7 +5,7 @@ import asyncio + import concurrent.futures + from pathlib import Path + +-import PyPDF2 ++import pypdf + + from .page_index import page_index + from .page_index_md import md_to_tree +@@ -79,7 +79,7 @@ class PageIndexClient: + # Extract per-page text so queries don't need the original PDF + pages = [] + with open(file_path, 'rb') as f: +- pdf_reader = PyPDF2.PdfReader(f) ++ pdf_reader = pypdf.PdfReader(f) + for i, page in enumerate(pdf_reader.pages, 1): + pages.append({'page': i, 'content': page.extract_text() or ''}) + diff --git a/pageindex/page_index.py b/pageindex/page_index.py index 083b26e..905ed25 100644 --- a/pageindex/page_index.py @@ -398,11 +420,31 @@ index 083b26e..905ed25 100644 else: toc_with_page_number = await meta_processor( page_list, +diff --git a/pageindex/retrieve.py b/pageindex/retrieve.py +index 55c3850..b6667a8 100644 +--- a/pageindex/retrieve.py ++++ b/pageindex/retrieve.py +@@ -1,5 +1,5 @@ + import json +-import PyPDF2 ++import pypdf + + try: + from .utils import get_number_of_pages, remove_fields +@@ -44,7 +44,7 @@ def _get_pdf_page_content(doc_info: dict, page_nums: list[int]) -> list[dict]: + ] + path = doc_info['path'] + with open(path, 'rb') as f: +- pdf_reader = PyPDF2.PdfReader(f) ++ pdf_reader = pypdf.PdfReader(f) + total = len(pdf_reader.pages) + valid_pages = [p for p in page_nums if 1 <= p <= total] + return [ diff --git a/pageindex/utils.py b/pageindex/utils.py -index 235dd09..39b0fb8 100644 +index 235dd09..bb35cb0 100644 --- a/pageindex/utils.py +++ b/pageindex/utils.py -@@ -1,6 +1,8 @@ +@@ -1,11 +1,13 @@ +import os +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") import litellm @@ -412,6 +454,12 @@ index 235dd09..39b0fb8 100644 import textwrap from datetime import datetime import time + import json +-import PyPDF2 ++import pypdf + import copy + import asyncio + import pymupdf @@ -30,59 +32,6 @@ def count_tokens(text, model=None): return litellm.token_counter(model=model, text=text) @@ -512,7 +560,13 @@ index 235dd09..39b0fb8 100644 def write_node_id(data, node_id=0): if isinstance(data, dict): data['node_id'] = str(node_id).zfill(4) -@@ -225,7 +141,7 @@ def extract_text_from_pdf(pdf_path): +@@ -220,26 +136,26 @@ def get_last_node(structure): + + + def extract_text_from_pdf(pdf_path): +- pdf_reader = PyPDF2.PdfReader(pdf_path) ++ pdf_reader = pypdf.PdfReader(pdf_path) + ###return text not list text="" for page_num in range(len(pdf_reader.pages)): page = pdf_reader.pages[page_num] @@ -521,7 +575,15 @@ index 235dd09..39b0fb8 100644 return text def get_pdf_title(pdf_path): -@@ -239,7 +155,7 @@ def get_text_of_pages(pdf_path, start_page, end_page, tag=True): +- pdf_reader = PyPDF2.PdfReader(pdf_path) ++ pdf_reader = pypdf.PdfReader(pdf_path) + meta = pdf_reader.metadata + title = meta.title if meta and meta.title else 'Untitled' + return title + + def get_text_of_pages(pdf_path, start_page, end_page, tag=True): +- pdf_reader = PyPDF2.PdfReader(pdf_path) ++ pdf_reader = pypdf.PdfReader(pdf_path) text = "" for page_num in range(start_page-1, end_page): page = pdf_reader.pages[page_num] @@ -530,7 +592,25 @@ index 235dd09..39b0fb8 100644 if tag: text += f"\n{page_text}\n\n" else: -@@ -391,7 +307,7 @@ def get_page_tokens(pdf_path, model=None, pdf_parser="PyPDF2"): +@@ -274,7 +190,7 @@ def get_pdf_name(pdf_path): + if isinstance(pdf_path, str): + pdf_name = os.path.basename(pdf_path) + elif isinstance(pdf_path, BytesIO): +- pdf_reader = PyPDF2.PdfReader(pdf_path) ++ pdf_reader = pypdf.PdfReader(pdf_path) + meta = pdf_reader.metadata + pdf_name = meta.title if meta and meta.title else 'Untitled' + pdf_name = sanitize_filename(pdf_name) +@@ -385,13 +301,13 @@ def add_preface_if_needed(data): + + + +-def get_page_tokens(pdf_path, model=None, pdf_parser="PyPDF2"): +- if pdf_parser == "PyPDF2": +- pdf_reader = PyPDF2.PdfReader(pdf_path) ++def get_page_tokens(pdf_path, model=None, pdf_parser="pypdf"): ++ if pdf_parser == "pypdf": ++ pdf_reader = pypdf.PdfReader(pdf_path) page_list = [] for page_num in range(len(pdf_reader.pages)): page = pdf_reader.pages[page_num] @@ -539,6 +619,15 @@ index 235dd09..39b0fb8 100644 token_length = litellm.token_counter(model=model, text=page_text) page_list.append((page_text, token_length)) return page_list +@@ -425,7 +341,7 @@ def get_text_of_pdf_pages_with_labels(pdf_pages, start_page, end_page): + return text + + def get_number_of_pages(pdf_path): +- pdf_reader = PyPDF2.PdfReader(pdf_path) ++ pdf_reader = pypdf.PdfReader(pdf_path) + num = len(pdf_reader.pages) + return num + @@ -576,10 +492,12 @@ def add_node_text_with_labels(node, pdf_pages): return @@ -590,3 +679,15 @@ index 235dd09..39b0fb8 100644 for line in text.splitlines(): print(textwrap.fill(line, width=width)) - +diff --git a/requirements.txt b/requirements.txt +index ae92bc4..e7f6dd7 100644 +--- a/requirements.txt ++++ b/requirements.txt +@@ -1,6 +1,6 @@ + litellm==1.84.0 + # openai-agents # optional: required for examples/agentic_vectorless_rag_demo.py + pymupdf==1.26.4 +-PyPDF2==3.0.1 ++pypdf==6.14.2 + python-dotenv==1.2.2 + pyyaml==6.0.2 diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index 7832a5e..7fb49c8 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "pydantic==2.13.4", "pydantic-settings==2.14.2", "pymupdf==1.28.0", - "pypdf2==3.0.1", + "pypdf==6.14.2", "python-dotenv==1.2.2", "python-multipart==0.0.32", "pyyaml==6.0.3", diff --git a/apps/api/scripts/install-pageindex.sh b/apps/api/scripts/install-pageindex.sh index 7b53d8d..0c97a38 100644 --- a/apps/api/scripts/install-pageindex.sh +++ b/apps/api/scripts/install-pageindex.sh @@ -2,7 +2,8 @@ set -eu destination="${1:-/opt/pageindex}" revision="190f8b378be58199ca993566a9214dba72089c54" -patch_file="$(dirname "$0")/../patches/pageindex-runtime.patch" +script_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +patch_file="${script_dir}/../patches/pageindex-runtime.patch" git clone --filter=blob:none https://github.com/VectifyAI/PageIndex.git "$destination" git -C "$destination" checkout "$revision" git -C "$destination" apply --check "$patch_file" diff --git a/apps/api/scripts/run_pageindex_pilot.py b/apps/api/scripts/run_pageindex_pilot.py index 68c6545..fba2d54 100644 --- a/apps/api/scripts/run_pageindex_pilot.py +++ b/apps/api/scripts/run_pageindex_pilot.py @@ -14,7 +14,7 @@ from decimal import Decimal from pathlib import Path -from PyPDF2 import PdfReader +from pypdf import PdfReader from sqlalchemy import func, select, text from vectorless_rag.config import Settings, get_settings diff --git a/apps/api/scripts/test_pageindex_patch.py b/apps/api/scripts/test_pageindex_patch.py index 43deceb..50316bc 100644 --- a/apps/api/scripts/test_pageindex_patch.py +++ b/apps/api/scripts/test_pageindex_patch.py @@ -6,6 +6,7 @@ import asyncio import importlib import sys +import tempfile from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -54,6 +55,36 @@ def test_page_grouping(page_index: Any) -> None: assert all("" in group for group in groups) +def test_pypdf_parser(utilities: Any) -> None: + def zero_tokens(**kwargs: object) -> int: + del kwargs + return 0 + + utilities.litellm.token_counter = zero_tokens + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "blank.pdf" + writer = utilities.pypdf.PdfWriter() + writer.add_blank_page(width=72, height=72) + with path.open("wb") as output: + writer.write(output) + + pages = utilities.get_page_tokens(path, model=None, pdf_parser="pypdf") + + assert pages == [("", 0)] + + +def test_pypdf_migration_is_complete(source: Path) -> None: + paths = ( + source / "pageindex/client.py", + source / "pageindex/retrieve.py", + source / "pageindex/utils.py", + source / "requirements.txt", + ) + + assert all("PyPDF2" not in path.read_text(encoding="utf-8") for path in paths) + assert "pypdf==6.14.2" in paths[-1].read_text(encoding="utf-8") + + def test_continuation_boundaries(page_index: Any) -> None: first = [{"structure": "1", "title": "Methods", "physical_index": 4}] assert page_index.merge_continuation_entries(first, list(first)) == first @@ -250,6 +281,8 @@ async def unreachable(*args: object, **kwargs: object) -> dict[str, Any]: def main() -> None: source = Path(sys.argv[1] if len(sys.argv) > 1 else "/opt/pageindex").resolve() + test_pypdf_migration_is_complete(source) + test_pypdf_parser(load_modules(source)[1]) test_page_grouping(load_modules(source)[0]) test_continuation_boundaries(load_modules(source)[0]) asyncio.run(test_large_node_keeps_first_subsection_at_node_start(load_modules(source)[0])) diff --git a/apps/api/src/vectorless_rag/config.py b/apps/api/src/vectorless_rag/config.py index 8ef3264..49f8c2a 100644 --- a/apps/api/src/vectorless_rag/config.py +++ b/apps/api/src/vectorless_rag/config.py @@ -32,8 +32,8 @@ class Settings(BaseSettings): pageindex_output_cost_per_million_usd: float | None = None pageindex_source_dir: Path = Path("/opt/pageindex") pageindex_work_dir: Path = Path("/workspaces") - pageindex_version: Literal["190f8b378be58199ca993566a9214dba72089c54+vr6"] = ( - "190f8b378be58199ca993566a9214dba72089c54+vr6" + pageindex_version: Literal["190f8b378be58199ca993566a9214dba72089c54+vr7"] = ( + "190f8b378be58199ca993566a9214dba72089c54+vr7" ) pageindex_llm_max_attempts: int = 3 pageindex_async_concurrency: int = 8 diff --git a/apps/api/src/vectorless_rag/pageindex_adapter.py b/apps/api/src/vectorless_rag/pageindex_adapter.py index 448eb1c..87e2c75 100644 --- a/apps/api/src/vectorless_rag/pageindex_adapter.py +++ b/apps/api/src/vectorless_rag/pageindex_adapter.py @@ -158,6 +158,8 @@ async def build_v2( await asyncio.to_thread(shutil.copyfile, pdf_path, local_pdf) preliminary_pages = await asyncio.to_thread(extract_pdf_pages, local_pdf) if len(preliminary_pages) == 1: + parser = "pymupdf" + parser_version = importlib.metadata.version("PyMuPDF") title = str(metadata.get("title") or "Document").strip() or "Document" summary = str( metadata.get("description") or metadata.get("abstract") or title @@ -176,6 +178,8 @@ async def build_v2( } pages = preliminary_pages else: + parser = "pypdf" + parser_version = importlib.metadata.version("pypdf") raw_tree, pages = await self._run_v2_child( workspace, local_pdf, @@ -202,8 +206,8 @@ async def build_v2( "add_doc_description": True, "add_node_text": False, }, - parser="PyPDF2", - parser_version=importlib.metadata.version("PyPDF2"), + parser=parser, + parser_version=parser_version, ) async def _run_v2_child( diff --git a/apps/api/src/vectorless_rag/run_pageindex.py b/apps/api/src/vectorless_rag/run_pageindex.py index 1484b0a..f10b556 100644 --- a/apps/api/src/vectorless_rag/run_pageindex.py +++ b/apps/api/src/vectorless_rag/run_pageindex.py @@ -71,7 +71,7 @@ def main() -> None: Callable[..., list[tuple[object, object]]], utilities.get_page_tokens, ) - pages = get_page_tokens(pdf_path, model=None, pdf_parser="PyPDF2") + pages = get_page_tokens(pdf_path, model=None, pdf_parser="pypdf") exact_text = [page[0] if isinstance(page[0], str) else "" for page in pages] known.pages_output.write_text( json.dumps(exact_text, ensure_ascii=False, separators=(",", ":")), diff --git a/apps/api/tests/test_config.py b/apps/api/tests/test_config.py index 2b15047..6128853 100644 --- a/apps/api/tests/test_config.py +++ b/apps/api/tests/test_config.py @@ -30,7 +30,7 @@ def test_ingestion_operational_defaults_are_bounded_and_patch_versioned() -> Non assert settings.pageindex_timeout_seconds == 3_600 assert settings.ingestion_max_attempts == 3 assert settings.ingestion_stale_grace_seconds == 300 - assert settings.pageindex_version.endswith("+vr6") + assert settings.pageindex_version.endswith("+vr7") assert settings.query_node_selection_max_output_tokens == 4_096 assert settings.request_deadline_seconds == 180 assert settings.catalog_routing_concurrency == 4 diff --git a/apps/api/tests/test_documentation_contracts.py b/apps/api/tests/test_documentation_contracts.py index db9d71b..1aa80f0 100644 --- a/apps/api/tests/test_documentation_contracts.py +++ b/apps/api/tests/test_documentation_contracts.py @@ -165,6 +165,31 @@ def test_license_and_release_metadata_are_consistent() -> None: assert "Copyright (c) 2026 ProofOfTechOrg" in license_text +def test_vulnerable_pdf_and_glob_dependencies_are_absent() -> None: + pyproject = tomllib.loads((ROOT / "apps/api/pyproject.toml").read_text(encoding="utf-8")) + uv_lock = (ROOT / "apps/api/uv.lock").read_text(encoding="utf-8").casefold() + pnpm_lock = (ROOT / "pnpm-lock.yaml").read_text(encoding="utf-8") + workspace = yaml.safe_load((ROOT / "pnpm-workspace.yaml").read_text(encoding="utf-8")) + + dependencies = [value.casefold() for value in pyproject["project"]["dependencies"]] + assert any(value.startswith("pypdf==") for value in dependencies) + assert all(not value.startswith("pypdf2") for value in dependencies) + assert '\nname = "pypdf2"\n' not in uv_lock + + brace_versions = [ + tuple(int(part) for part in version.split(".")) + for version in re.findall( + r"^ brace-expansion@(\d+\.\d+\.\d+):$", pnpm_lock, flags=re.MULTILINE + ) + ] + override = tuple( + int(part) for part in str(workspace["overrides"]["brace-expansion"]).split(".") + ) + assert brace_versions + assert all(version >= (5, 0, 8) for version in brace_versions) + assert override >= (5, 0, 8) + + def test_provider_price_and_retry_settings_reach_deployments() -> None: environment = (ROOT / ".env.example").read_text(encoding="utf-8") compose = (ROOT / "compose.yaml").read_text(encoding="utf-8") diff --git a/apps/api/tests/test_operator_scripts.py b/apps/api/tests/test_operator_scripts.py index 56c8cc5..737c502 100644 --- a/apps/api/tests/test_operator_scripts.py +++ b/apps/api/tests/test_operator_scripts.py @@ -6,6 +6,7 @@ import json import os import re +import shutil import stat import subprocess import sys @@ -25,6 +26,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker API_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = API_ROOT.parents[1] sys.path.insert(0, str(API_ROOT)) from scripts import ( # noqa: E402 @@ -98,6 +100,43 @@ def test_pilot_schema_guard_matches_alembic_head() -> None: ) +def test_pageindex_installer_resolves_patch_from_relative_script_path(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_git = fake_bin / "git" + fake_git.write_text( + """#!/usr/bin/env python3 +import sys +from pathlib import Path + +arguments = sys.argv[1:] +if arguments[0] == "clone": + Path(arguments[-1]).mkdir(parents=True) +elif arguments[0] == "-C" and "apply" in arguments: + patch = Path(arguments[-1]) + if not patch.is_absolute() or not patch.is_file(): + raise SystemExit(42) +""", + encoding="utf-8", + ) + fake_git.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + script = API_ROOT / "scripts/install-pageindex.sh" + shell = shutil.which("sh") + assert shell is not None + environment = {**os.environ, "PATH": f"{fake_bin}{os.pathsep}{os.environ['PATH']}"} + + result = subprocess.run( # noqa: S603 + [shell, str(script.relative_to(REPOSITORY_ROOT)), str(tmp_path / "pageindex")], + cwd=REPOSITORY_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + + async def _add_indexed_document( session: AsyncSession, *, diff --git a/apps/api/tests/test_pageindex_adapter.py b/apps/api/tests/test_pageindex_adapter.py index 766154b..592cc09 100644 --- a/apps/api/tests/test_pageindex_adapter.py +++ b/apps/api/tests/test_pageindex_adapter.py @@ -1,4 +1,5 @@ import asyncio +import importlib.metadata import json import uuid from pathlib import Path @@ -232,6 +233,61 @@ async def fail_subprocess(*args: object, **kwargs: object) -> None: assert artifact["pages"] == [{"page": 1, "text": "unique one-page evidence"}] +@pytest.mark.parametrize( + ("preliminary_pages", "expected_parser", "distribution"), + [ + (["unique one-page evidence"], "pymupdf", "PyMuPDF"), + (["one", "two"], "pypdf", "pypdf"), + ], +) +async def test_v2_artifact_records_actual_parser( + monkeypatch: MonkeyPatch, + tmp_path: Path, + preliminary_pages: list[str], + expected_parser: str, + distribution: str, +) -> None: + source = tmp_path / "source.pdf" + source.write_bytes(b"%PDF-test") + sentinel = object() + captured: dict[str, Any] = {} + + async def inline_to_thread(function: Any, *args: Any, **kwargs: Any) -> Any: + return function(*args, **kwargs) + + def extracted_pages(path: Path) -> list[str]: + del path + return preliminary_pages + + async def run_child(*args: object, **kwargs: object) -> tuple[object, list[str]]: + del args, kwargs + return {"doc_description": "summary", "structure": []}, preliminary_pages + + def capture_build(*args: object, **kwargs: Any) -> Any: + del args + captured.update(kwargs) + return sentinel + + monkeypatch.setattr("vectorless_rag.pageindex_adapter.asyncio.to_thread", inline_to_thread) + monkeypatch.setattr("vectorless_rag.pageindex_adapter.extract_pdf_pages", extracted_pages) + monkeypatch.setattr("vectorless_rag.pageindex_adapter.build_manifest_v2", capture_build) + monkeypatch.setattr(PageIndexAdapter, "_run_v2_child", run_child) + adapter = PageIndexAdapter(Settings(pageindex_work_dir=tmp_path / "work")) + + result = await adapter.build_v2( + source, + "document-id", + "0" * 64, + {"title": "One page"}, + ingestion_attempt_id=_ATTEMPT_ID, + trace_id=_TRACE_ID, + ) + + assert result is sentinel + assert captured["parser"] == expected_parser + assert captured["parser_version"] == importlib.metadata.version(distribution) + + async def test_subprocess_success_reads_only_the_artifact( monkeypatch: MonkeyPatch, tmp_path: Path ) -> None: diff --git a/apps/api/tests/test_pageindex_artifact_v2.py b/apps/api/tests/test_pageindex_artifact_v2.py index 237d484..ec08fed 100644 --- a/apps/api/tests/test_pageindex_artifact_v2.py +++ b/apps/api/tests/test_pageindex_artifact_v2.py @@ -376,8 +376,8 @@ def test_a_page_containing_a_special_token_literal_still_indexes() -> None: model="model", prompt_profile="profile", options_profile={}, - parser="PyPDF2", - parser_version="3.0.0", + parser="pypdf", + parser_version="6.14.2", ) assert build.manifest.quality.page_count == 10 diff --git a/apps/api/uv.lock b/apps/api/uv.lock index 26e7063..2c810cf 100644 --- a/apps/api/uv.lock +++ b/apps/api/uv.lock @@ -3,7 +3,7 @@ revision = 3 requires-python = "==3.12.*" [options] -exclude-newer = "2026-07-13T08:43:23.631009754Z" +exclude-newer = "2026-07-24T06:37:14.540488537Z" exclude-newer-span = "P7D" [options.exclude-newer-package] @@ -1255,12 +1255,12 @@ wheels = [ ] [[package]] -name = "pypdf2" -version = "3.0.1" +name = "pypdf" +version = "6.14.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/bb/18dc3062d37db6c491392007dfd1a7f524bb95886eb956569ac38a23a784/PyPDF2-3.0.1.tar.gz", hash = "sha256:a74408f69ba6271f71b9352ef4ed03dc53a31aa404d29b5d31f53bfecfee1440", size = 227419, upload-time = "2022-12-31T10:36:13.13Z" } +sdist = { url = "https://files.pythonhosted.org/packages/03/72/7dfd5ff1c9c37de97a731701f51af091325f123d9d4270361c9c69e4431f/pypdf-6.14.2.tar.gz", hash = "sha256:7873f502fe4385e79539b21d872392dc0c4e3714327c15881cbc7fbfd1f95b25", size = 6491182, upload-time = "2026-06-23T14:18:30.859Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/5e/c86a5643653825d3c913719e788e41386bee415c2b87b4f955432f2de6b2/pypdf2-3.0.1-py3-none-any.whl", hash = "sha256:d16e4205cfee272fbdc0568b68d82be796540b1537508cef59388f839c191928", size = 232572, upload-time = "2022-12-31T10:36:10.327Z" }, + { url = "https://files.pythonhosted.org/packages/49/e6/136aa8993a2ae7214e0b0ef2edaa0d2e08d1d4e4982635b08a835ff31ec8/pypdf-6.14.2-py3-none-any.whl", hash = "sha256:3f07891af76dc002657e04993ab9b4de81de29f9013b9761d0b7968bff12e946", size = 349514, upload-time = "2026-06-23T14:18:28.867Z" }, ] [[package]] @@ -1824,7 +1824,7 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "pymupdf" }, - { name = "pypdf2" }, + { name = "pypdf" }, { name = "python-dotenv" }, { name = "python-multipart" }, { name = "pyyaml" }, @@ -1864,7 +1864,7 @@ requires-dist = [ { name = "pydantic", specifier = "==2.13.4" }, { name = "pydantic-settings", specifier = "==2.14.2" }, { name = "pymupdf", specifier = "==1.28.0" }, - { name = "pypdf2", specifier = "==3.0.1" }, + { name = "pypdf", specifier = "==6.14.2" }, { name = "python-dotenv", specifier = "==1.2.2" }, { name = "python-multipart", specifier = "==0.0.32" }, { name = "pyyaml", specifier = "==6.0.3" }, diff --git a/docs/pageindex-runtime-patch.md b/docs/pageindex-runtime-patch.md index 1fcc8fb..796cf0a 100644 --- a/docs/pageindex-runtime-patch.md +++ b/docs/pageindex-runtime-patch.md @@ -3,7 +3,7 @@ # PageIndex runtime patch -Reference for `apps/api/patches/pageindex-runtime.patch`: every change it makes to the vendored PageIndex source, and whether each one fixes an upstream bug or adds something the ingestion pipeline needs. The patch applies to [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex) at pinned revision `190f8b378be58199ca993566a9214dba72089c54` and touches two files: `pageindex/page_index.py` and `pageindex/utils.py`. The current patch revision is `+vr6`. +Reference for `apps/api/patches/pageindex-runtime.patch`: every change it makes to the vendored PageIndex source, and whether each one fixes an upstream bug or adds something the ingestion pipeline needs. The patch applies to [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex) at pinned revision `190f8b378be58199ca993566a9214dba72089c54` and touches `pageindex/client.py`, `pageindex/page_index.py`, `pageindex/retrieve.py`, `pageindex/utils.py`, and `requirements.txt`. The current patch revision is `+vr7`. ## How the patch is applied and versioned @@ -11,7 +11,7 @@ Reference for `apps/api/patches/pageindex-runtime.patch`: every change it makes Three identity surfaces track the patch: -- **`pageindex_version`**: pinned as the literal `190f8b378be58199ca993566a9214dba72089c54+vr6` in `config.py`. Bump the `+vrN` suffix whenever patch behavior changes. +- **`pageindex_version`**: pinned as the literal `190f8b378be58199ca993566a9214dba72089c54+vr7` in `config.py`. Bump the `+vrN` suffix whenever patch behavior changes. - **`patch_sha256`**: computed from the patch file at ingestion time in `pageindex_adapter.py`, so any byte change to the patch shifts artifact identity even without a version bump. - **`configuration_hash` and `recipe_hash`**: both digest the full version string and the patch digest, so a rebuilt document under a new patch never reuses an old artifact. @@ -34,7 +34,7 @@ Each change is either a fix for a defect in upstream PageIndex or a feature the | `process_none_page_numbers` robustness | `page_index.py` | Crash fixes | | Large-node recursion repair | `page_index.py` | Bug fixes | | Bottom-up direct and subtree summaries | `utils.py` | Missing feature | -| PyPDF2 empty-page guards | `utils.py` | Crash fix | +| `pypdf` migration and empty-page guards | `client.py`, `retrieve.py`, `utils.py`, `requirements.txt` | Security and crash fix | | litellm cost-map environment default | `utils.py` | Integration nit | ## Model-call plumbing (`utils.py`) @@ -128,9 +128,9 @@ The v2 artifact distinguishes a node's own content from its subtree. Upstream su The patched `generate_summaries_for_structure` walks each tree bottom-up: it summarizes children first, then produces both a `direct_summary` (the node alone) and a `subtree_summary` (the node with its children's summaries folded into the prompt), storing the subtree summary as `summary` for compatibility. `generate_node_summary` accepts the child summaries and tolerates missing text. Kind: missing feature; the artifact schema's `direct_summary` and `subtree_summary` fields depend on it. -## PyPDF2 empty-page guards (`utils.py`) +## `pypdf` migration and empty-page guards (`utils.py`) -`PyPDF2`'s `extract_text()` returns `None` for pages without extractable text, and upstream concatenated that `None` into a string in three places (`extract_text_from_pdf`, `get_text_of_pages`, `get_page_tokens`). Each site now appends `page.extract_text() or ""`. Kind: crash fix. +The patch replaces unmaintained `PyPDF2` with maintained `pypdf` in `client.py`, `retrieve.py`, `utils.py`, and `requirements.txt`. Multi-page artifacts generated through PageIndex record `pypdf` and its installed version in artifact identity; the single-page shortcut records the PyMuPDF parser it actually uses. The parser’s `extract_text()` can return `None` for pages without extractable text, and upstream concatenated that `None` into a string in three places (`extract_text_from_pdf`, `get_text_of_pages`, `get_page_tokens`). Each site now appends `page.extract_text() or ""`. Kind: security and crash fix. ## litellm cost-map default (`utils.py`) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 190e345..2e01d64 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,9 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + brace-expansion: 5.0.8 + importers: .: @@ -1221,9 +1224,9 @@ packages: bidi-js@1.0.3: resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==} - brace-expansion@5.0.7: - resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} - engines: {node: 18 || 20 || >=22} + brace-expansion@5.0.8: + resolution: {integrity: sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==} + engines: {node: 20 || >=22} browserslist@4.28.6: resolution: {integrity: sha512-FQBYNK15VMslhLHpA7+n+n1GOlF1kId2xcCg7/j95f24AOF6VDYMNH4mFxF7KuaTdv627faazpOAjFzMrfJOUw==} @@ -3493,7 +3496,7 @@ snapshots: dependencies: require-from-string: 2.0.2 - brace-expansion@5.0.7: + brace-expansion@5.0.8: dependencies: balanced-match: 4.0.4 @@ -4004,7 +4007,7 @@ snapshots: minimatch@10.2.5: dependencies: - brace-expansion: 5.0.7 + brace-expansion: 5.0.8 ms@2.1.3: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2483565..787efa1 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,9 @@ packages: - apps/web +overrides: + brace-expansion: 5.0.8 + minimumReleaseAge: 10080 minimumReleaseAgeExclude: - "@astryxdesign/cli@0.1.6"