Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ This changelog records user-visible changes. Vectorless RAG follows semantic ver

## Unreleased

No changes have been recorded after v0.3.0.
### Fixed

- Replaced unmaintained `PyPDF2` with maintained `pypdf` across the API and vendored PageIndex runtime
- Updated the vulnerable transitive `brace-expansion` package to its quarantined patched release
- Made the PageIndex installer resolve its patch path correctly when invoked through a relative path

## 0.3.0, 2026-07-31

Expand Down
111 changes: 106 additions & 5 deletions apps/api/patches/pageindex-runtime.patch
Original file line number Diff line number Diff line change
@@ -1,3 +1,25 @@
diff --git a/pageindex/client.py b/pageindex/client.py
index 894dab1..ec5ec3b 100644
--- a/pageindex/client.py
+++ b/pageindex/client.py
@@ -5,7 +5,7 @@ import asyncio
import concurrent.futures
from pathlib import Path

-import PyPDF2
+import pypdf

from .page_index import page_index
from .page_index_md import md_to_tree
@@ -79,7 +79,7 @@ class PageIndexClient:
# Extract per-page text so queries don't need the original PDF
pages = []
with open(file_path, 'rb') as f:
- pdf_reader = PyPDF2.PdfReader(f)
+ pdf_reader = pypdf.PdfReader(f)
for i, page in enumerate(pdf_reader.pages, 1):
pages.append({'page': i, 'content': page.extract_text() or ''})

diff --git a/pageindex/page_index.py b/pageindex/page_index.py
index 083b26e..905ed25 100644
--- a/pageindex/page_index.py
Expand Down Expand Up @@ -398,11 +420,31 @@ index 083b26e..905ed25 100644
else:
toc_with_page_number = await meta_processor(
page_list,
diff --git a/pageindex/retrieve.py b/pageindex/retrieve.py
index 55c3850..b6667a8 100644
--- a/pageindex/retrieve.py
+++ b/pageindex/retrieve.py
@@ -1,5 +1,5 @@
import json
-import PyPDF2
+import pypdf

try:
from .utils import get_number_of_pages, remove_fields
@@ -44,7 +44,7 @@ def _get_pdf_page_content(doc_info: dict, page_nums: list[int]) -> list[dict]:
]
path = doc_info['path']
with open(path, 'rb') as f:
- pdf_reader = PyPDF2.PdfReader(f)
+ pdf_reader = pypdf.PdfReader(f)
total = len(pdf_reader.pages)
valid_pages = [p for p in page_nums if 1 <= p <= total]
return [
diff --git a/pageindex/utils.py b/pageindex/utils.py
index 235dd09..39b0fb8 100644
index 235dd09..bb35cb0 100644
--- a/pageindex/utils.py
+++ b/pageindex/utils.py
@@ -1,6 +1,8 @@
@@ -1,11 +1,13 @@
+import os
+os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True")
import litellm
Expand All @@ -412,6 +454,12 @@ index 235dd09..39b0fb8 100644
import textwrap
from datetime import datetime
import time
import json
-import PyPDF2
+import pypdf
import copy
import asyncio
import pymupdf
@@ -30,59 +32,6 @@ def count_tokens(text, model=None):
return litellm.token_counter(model=model, text=text)

Expand Down Expand Up @@ -512,7 +560,13 @@ index 235dd09..39b0fb8 100644
def write_node_id(data, node_id=0):
if isinstance(data, dict):
data['node_id'] = str(node_id).zfill(4)
@@ -225,7 +141,7 @@ def extract_text_from_pdf(pdf_path):
@@ -220,26 +136,26 @@ def get_last_node(structure):


def extract_text_from_pdf(pdf_path):
- pdf_reader = PyPDF2.PdfReader(pdf_path)
+ pdf_reader = pypdf.PdfReader(pdf_path)
###return text not list
text=""
for page_num in range(len(pdf_reader.pages)):
page = pdf_reader.pages[page_num]
Expand All @@ -521,7 +575,15 @@ index 235dd09..39b0fb8 100644
return text

def get_pdf_title(pdf_path):
@@ -239,7 +155,7 @@ def get_text_of_pages(pdf_path, start_page, end_page, tag=True):
- pdf_reader = PyPDF2.PdfReader(pdf_path)
+ pdf_reader = pypdf.PdfReader(pdf_path)
meta = pdf_reader.metadata
title = meta.title if meta and meta.title else 'Untitled'
return title

def get_text_of_pages(pdf_path, start_page, end_page, tag=True):
- pdf_reader = PyPDF2.PdfReader(pdf_path)
+ pdf_reader = pypdf.PdfReader(pdf_path)
text = ""
for page_num in range(start_page-1, end_page):
page = pdf_reader.pages[page_num]
Expand All @@ -530,7 +592,25 @@ index 235dd09..39b0fb8 100644
if tag:
text += f"<start_index_{page_num+1}>\n{page_text}\n<end_index_{page_num+1}>\n"
else:
@@ -391,7 +307,7 @@ def get_page_tokens(pdf_path, model=None, pdf_parser="PyPDF2"):
@@ -274,7 +190,7 @@ def get_pdf_name(pdf_path):
if isinstance(pdf_path, str):
pdf_name = os.path.basename(pdf_path)
elif isinstance(pdf_path, BytesIO):
- pdf_reader = PyPDF2.PdfReader(pdf_path)
+ pdf_reader = pypdf.PdfReader(pdf_path)
meta = pdf_reader.metadata
pdf_name = meta.title if meta and meta.title else 'Untitled'
pdf_name = sanitize_filename(pdf_name)
@@ -385,13 +301,13 @@ def add_preface_if_needed(data):



-def get_page_tokens(pdf_path, model=None, pdf_parser="PyPDF2"):
- if pdf_parser == "PyPDF2":
- pdf_reader = PyPDF2.PdfReader(pdf_path)
+def get_page_tokens(pdf_path, model=None, pdf_parser="pypdf"):
+ if pdf_parser == "pypdf":
+ pdf_reader = pypdf.PdfReader(pdf_path)
page_list = []
for page_num in range(len(pdf_reader.pages)):
page = pdf_reader.pages[page_num]
Expand All @@ -539,6 +619,15 @@ index 235dd09..39b0fb8 100644
token_length = litellm.token_counter(model=model, text=page_text)
page_list.append((page_text, token_length))
return page_list
@@ -425,7 +341,7 @@ def get_text_of_pdf_pages_with_labels(pdf_pages, start_page, end_page):
return text

def get_number_of_pages(pdf_path):
- pdf_reader = PyPDF2.PdfReader(pdf_path)
+ pdf_reader = pypdf.PdfReader(pdf_path)
num = len(pdf_reader.pages)
return num

@@ -576,10 +492,12 @@ def add_node_text_with_labels(node, pdf_pages):
return

Expand Down Expand Up @@ -590,3 +679,15 @@ index 235dd09..39b0fb8 100644
for line in text.splitlines():
print(textwrap.fill(line, width=width))
-
diff --git a/requirements.txt b/requirements.txt
index ae92bc4..e7f6dd7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
litellm==1.84.0
# openai-agents # optional: required for examples/agentic_vectorless_rag_demo.py
pymupdf==1.26.4
-PyPDF2==3.0.1
+pypdf==6.14.2
python-dotenv==1.2.2
pyyaml==6.0.2
2 changes: 1 addition & 1 deletion apps/api/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ dependencies = [
"pydantic==2.13.4",
"pydantic-settings==2.14.2",
"pymupdf==1.28.0",
"pypdf2==3.0.1",
"pypdf==6.14.2",
"python-dotenv==1.2.2",
"python-multipart==0.0.32",
"pyyaml==6.0.3",
Expand Down
3 changes: 2 additions & 1 deletion apps/api/scripts/install-pageindex.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
set -eu
destination="${1:-/opt/pageindex}"
revision="190f8b378be58199ca993566a9214dba72089c54"
patch_file="$(dirname "$0")/../patches/pageindex-runtime.patch"
script_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
patch_file="${script_dir}/../patches/pageindex-runtime.patch"
git clone --filter=blob:none https://github.com/VectifyAI/PageIndex.git "$destination"
git -C "$destination" checkout "$revision"
git -C "$destination" apply --check "$patch_file"
Expand Down
2 changes: 1 addition & 1 deletion apps/api/scripts/run_pageindex_pilot.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from decimal import Decimal
from pathlib import Path

from PyPDF2 import PdfReader
from pypdf import PdfReader
from sqlalchemy import func, select, text

from vectorless_rag.config import Settings, get_settings
Expand Down
33 changes: 33 additions & 0 deletions apps/api/scripts/test_pageindex_patch.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import asyncio
import importlib
import sys
import tempfile
from pathlib import Path
from types import SimpleNamespace
from typing import Any, cast
Expand Down Expand Up @@ -54,6 +55,36 @@ def test_page_grouping(page_index: Any) -> None:
assert all("<physical_index_1>" in group for group in groups)


def test_pypdf_parser(utilities: Any) -> None:
def zero_tokens(**kwargs: object) -> int:
del kwargs
return 0

utilities.litellm.token_counter = zero_tokens
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "blank.pdf"
writer = utilities.pypdf.PdfWriter()
writer.add_blank_page(width=72, height=72)
with path.open("wb") as output:
writer.write(output)

pages = utilities.get_page_tokens(path, model=None, pdf_parser="pypdf")

assert pages == [("", 0)]


def test_pypdf_migration_is_complete(source: Path) -> None:
paths = (
source / "pageindex/client.py",
source / "pageindex/retrieve.py",
source / "pageindex/utils.py",
source / "requirements.txt",
)

assert all("PyPDF2" not in path.read_text(encoding="utf-8") for path in paths)
assert "pypdf==6.14.2" in paths[-1].read_text(encoding="utf-8")


def test_continuation_boundaries(page_index: Any) -> None:
first = [{"structure": "1", "title": "Methods", "physical_index": 4}]
assert page_index.merge_continuation_entries(first, list(first)) == first
Expand Down Expand Up @@ -250,6 +281,8 @@ async def unreachable(*args: object, **kwargs: object) -> dict[str, Any]:

def main() -> None:
source = Path(sys.argv[1] if len(sys.argv) > 1 else "/opt/pageindex").resolve()
test_pypdf_migration_is_complete(source)
test_pypdf_parser(load_modules(source)[1])
test_page_grouping(load_modules(source)[0])
test_continuation_boundaries(load_modules(source)[0])
asyncio.run(test_large_node_keeps_first_subsection_at_node_start(load_modules(source)[0]))
Expand Down
4 changes: 2 additions & 2 deletions apps/api/src/vectorless_rag/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ class Settings(BaseSettings):
pageindex_output_cost_per_million_usd: float | None = None
pageindex_source_dir: Path = Path("/opt/pageindex")
pageindex_work_dir: Path = Path("/workspaces")
pageindex_version: Literal["190f8b378be58199ca993566a9214dba72089c54+vr6"] = (
"190f8b378be58199ca993566a9214dba72089c54+vr6"
pageindex_version: Literal["190f8b378be58199ca993566a9214dba72089c54+vr7"] = (
"190f8b378be58199ca993566a9214dba72089c54+vr7"
)
pageindex_llm_max_attempts: int = 3
pageindex_async_concurrency: int = 8
Expand Down
8 changes: 6 additions & 2 deletions apps/api/src/vectorless_rag/pageindex_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,8 @@ async def build_v2(
await asyncio.to_thread(shutil.copyfile, pdf_path, local_pdf)
preliminary_pages = await asyncio.to_thread(extract_pdf_pages, local_pdf)
if len(preliminary_pages) == 1:
parser = "pymupdf"
parser_version = importlib.metadata.version("PyMuPDF")
title = str(metadata.get("title") or "Document").strip() or "Document"
summary = str(
metadata.get("description") or metadata.get("abstract") or title
Expand All @@ -176,6 +178,8 @@ async def build_v2(
}
pages = preliminary_pages
else:
parser = "pypdf"
parser_version = importlib.metadata.version("pypdf")
raw_tree, pages = await self._run_v2_child(
workspace,
local_pdf,
Expand All @@ -202,8 +206,8 @@ async def build_v2(
"add_doc_description": True,
"add_node_text": False,
},
parser="PyPDF2",
parser_version=importlib.metadata.version("PyPDF2"),
parser=parser,
parser_version=parser_version,
)

async def _run_v2_child(
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/vectorless_rag/run_pageindex.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def main() -> None:
Callable[..., list[tuple[object, object]]],
utilities.get_page_tokens,
)
pages = get_page_tokens(pdf_path, model=None, pdf_parser="PyPDF2")
pages = get_page_tokens(pdf_path, model=None, pdf_parser="pypdf")
exact_text = [page[0] if isinstance(page[0], str) else "" for page in pages]
known.pages_output.write_text(
json.dumps(exact_text, ensure_ascii=False, separators=(",", ":")),
Expand Down
2 changes: 1 addition & 1 deletion apps/api/tests/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def test_ingestion_operational_defaults_are_bounded_and_patch_versioned() -> Non
assert settings.pageindex_timeout_seconds == 3_600
assert settings.ingestion_max_attempts == 3
assert settings.ingestion_stale_grace_seconds == 300
assert settings.pageindex_version.endswith("+vr6")
assert settings.pageindex_version.endswith("+vr7")
assert settings.query_node_selection_max_output_tokens == 4_096
assert settings.request_deadline_seconds == 180
assert settings.catalog_routing_concurrency == 4
Expand Down
25 changes: 25 additions & 0 deletions apps/api/tests/test_documentation_contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,31 @@ def test_license_and_release_metadata_are_consistent() -> None:
assert "Copyright (c) 2026 ProofOfTechOrg" in license_text


def test_vulnerable_pdf_and_glob_dependencies_are_absent() -> None:
pyproject = tomllib.loads((ROOT / "apps/api/pyproject.toml").read_text(encoding="utf-8"))
uv_lock = (ROOT / "apps/api/uv.lock").read_text(encoding="utf-8").casefold()
pnpm_lock = (ROOT / "pnpm-lock.yaml").read_text(encoding="utf-8")
workspace = yaml.safe_load((ROOT / "pnpm-workspace.yaml").read_text(encoding="utf-8"))

dependencies = [value.casefold() for value in pyproject["project"]["dependencies"]]
assert any(value.startswith("pypdf==") for value in dependencies)
assert all(not value.startswith("pypdf2") for value in dependencies)
assert '\nname = "pypdf2"\n' not in uv_lock

brace_versions = [
tuple(int(part) for part in version.split("."))
for version in re.findall(
r"^ brace-expansion@(\d+\.\d+\.\d+):$", pnpm_lock, flags=re.MULTILINE
)
]
override = tuple(
int(part) for part in str(workspace["overrides"]["brace-expansion"]).split(".")
)
assert brace_versions
assert all(version >= (5, 0, 8) for version in brace_versions)
assert override >= (5, 0, 8)


def test_provider_price_and_retry_settings_reach_deployments() -> None:
environment = (ROOT / ".env.example").read_text(encoding="utf-8")
compose = (ROOT / "compose.yaml").read_text(encoding="utf-8")
Expand Down
Loading
Loading