diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a55ec35 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.github +.venv +.cache +.env +**/__pycache__ +.pytest_cache +.ruff_cache +tests +deploy +dist +build diff --git a/.env.example b/.env.example index 146e569..21808ad 100644 --- a/.env.example +++ b/.env.example @@ -10,3 +10,12 @@ CHAPTERWISE_CACHE_DIR=.cache/chapterwise # Sections longer than this (approx. tokens) are summarized in parts, then merged CHAPTERWISE_MAX_CHUNK_TOKENS=24000 + +# --- Service (web UI + worker) ------------------------------------------------ +DATABASE_URL=postgresql+psycopg://chapterwise:chapterwise@localhost:5432/chapterwise +CHAPTERWISE_BOOKS_DIR=./books +CHAPTERWISE_SECRET_KEY=change-me-to-a-long-random-string +CHAPTERWISE_ADMIN_PASSWORD=change-me +CHAPTERWISE_PUBLIC_URL=http://localhost:8100 +# Seconds between automatic scans of the books folder +CHAPTERWISE_SCAN_INTERVAL=600 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3ad6b9..86c695d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,21 +1,39 @@ -name: CI +name: CI/CD + +# Every push and PR: lint + tests (with a real Postgres). +# Pushes to main: build the image, push it to GHCR, deploy it to the VPS. on: push: branches: [main] pull_request: + workflow_dispatch: permissions: contents: read +env: + IMAGE: ghcr.io/${{ github.repository }} + jobs: test: runs-on: ubuntu-latest strategy: matrix: python: ["3.12", "3.13"] + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: chapterwise_test + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 + env: + TEST_DATABASE_URL: postgresql+psycopg://postgres:postgres@localhost:5432/chapterwise_test steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -34,4 +52,114 @@ jobs: ruff format --check . - name: Test - run: pytest + run: pytest -rs + + build: + needs: test + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + push: true + platforms: linux/amd64 + build-args: VERSION=${{ github.sha }} + tags: | + ${{ env.IMAGE }}:sha-${{ github.sha }} + ${{ env.IMAGE }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + deploy: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + packages: read + concurrency: + group: deploy-chapterwise + cancel-in-progress: false + env: + REMOTE: root@${{ secrets.DEPLOY_HOST }} + REMOTE_DIR: /root/chapterwise + TAG: sha-${{ github.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check deploy secrets + env: + HOST: ${{ secrets.DEPLOY_HOST }} + KNOWN: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + run: | + if [ -z "$HOST" ] || [ -z "$KNOWN" ]; then + echo "::error::Set the DEPLOY_HOST, DEPLOY_KNOWN_HOSTS and DEPLOY_SSH_KEY secrets (see README)." + exit 1 + fi + + - uses: webfactory/ssh-agent@a6f90b1f127823b31d4d4a8d96047790581349bd # v0.9.1 + with: + ssh-private-key: ${{ secrets.DEPLOY_SSH_KEY }} + + # Pinned host key instead of ssh-keyscan, which would trust whoever answers on that address. + - name: Trust the server's host key + env: + KNOWN: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + run: mkdir -p ~/.ssh && echo "$KNOWN" >> ~/.ssh/known_hosts + + - name: Check the server is bootstrapped + run: | + if ! ssh "$REMOTE" "test -f $REMOTE_DIR/.env"; then + echo "::error::$REMOTE_DIR/.env missing: run deploy/server-bootstrap.sh on the VPS first." + exit 1 + fi + + - name: Upload config + run: rsync -az --chmod=F644 deploy/docker-compose.yml "$REMOTE:$REMOTE_DIR/" && + rsync -az --chmod=F755 deploy/cw.sh "$REMOTE:$REMOTE_DIR/" + + - name: Back up the database + run: ssh "$REMOTE" "cd $REMOTE_DIR && touch image.env && ./cw.sh backup pre-$TAG" + + - name: Pull and start + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # The token goes through stdin, never onto a command line, and is logged out right after the pull. + echo "$GH_TOKEN" | ssh "$REMOTE" "docker login ghcr.io -u ${{ github.actor }} --password-stdin" + ssh "$REMOTE" "cd $REMOTE_DIR \ + && echo IMAGE=$IMAGE:$TAG > image.env \ + && (./cw.sh pull; status=\$?; docker logout ghcr.io; exit \$status) \ + && ./cw.sh up -d --remove-orphans" + + - name: Verify health + run: | + for _ in $(seq 1 30); do + if ssh "$REMOTE" 'curl -sf http://127.0.0.1:8100/healthz >/dev/null'; then + echo "Healthy on $TAG" + ssh "$REMOTE" "echo $TAG > $REMOTE_DIR/.deployed && docker image prune -f >/dev/null" + exit 0 + fi + sleep 5 + done + echo "::error::Health check failed. Recent logs:" + ssh "$REMOTE" "cd $REMOTE_DIR && ./cw.sh ps && ./cw.sh logs --tail 80 migrate web worker" || true + exit 1 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3a6e8ab --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +# One image for every role: `chapterwise serve` (default), `chapterwise worker`, `chapterwise migrate`. + +FROM python:3.12-slim AS build +WORKDIR /src +COPY pyproject.toml README.md LICENSE ./ +COPY src ./src +RUN pip wheel --no-cache-dir --wheel-dir /wheels . + +FROM python:3.12-slim +ARG VERSION=dev +LABEL org.opencontainers.image.source="https://github.com/aminghuf/chapterwise" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.licenses="MIT" +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + CHAPTERWISE_BOOKS_DIR=/books +RUN useradd --uid 1000 --create-home app +COPY --from=build /wheels /wheels +RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels +USER app +WORKDIR /home/app +EXPOSE 8100 +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8100/healthz', timeout=4)"] +CMD ["chapterwise", "serve", "--host", "0.0.0.0", "--port", "8100"] diff --git a/README.md b/README.md index 1146176..3aeaf32 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,16 @@ chapterwise splits a book into sections using its own table of contents, summari same request is never paid for twice. It's built to run next to a self-hosted library such as [BookOrbit](https://github.com/bookorbit/bookorbit), reading the same books folder read-only. -**Status:** Phase 1, a command-line tool. The web UI, Telegram delivery and scheduling come next -(see [Roadmap](#roadmap)). +**Status:** Phase 2. A command-line tool plus a self-hosted web app (library, chapters, summaries) +with a background worker, deployed by CI/CD. Telegram delivery comes next (see [Roadmap](#roadmap)). + +``` +books folder (read-only) ──► worker: scan · parse · summarize ──► DeepSeek API + │ + Postgres (books, sections, job queue, LLM cache) + │ + browser ──► Caddy ──► web: login · library · book · section +``` ## Supported formats @@ -52,6 +60,53 @@ Summaries are written in the same language as the book. are free, an interrupted run resumes where it stopped, and changing the model or a prompt regenerates only what's affected. +## The service + +Three processes from one image or install, sharing one Postgres database: + +| Command | Role | +| --- | --- | +| `chapterwise migrate` | Create or upgrade the schema (Alembic) | +| `chapterwise serve` | Web UI on port 8100 | +| `chapterwise worker` | Scans the books folder every `CHAPTERWISE_SCAN_INTERVAL` seconds, parses new or changed files, runs summaries | + +- **Job queue:** a Postgres table claimed with `FOR UPDATE SKIP LOCKED`, so there's no Redis to run. +- **Retries:** failed jobs retry with exponential backoff, up to 3 attempts. Jobs interrupted by a + restart are requeued. +- **Summaries are cached** in Postgres, keyed like the CLI cache. +- **Web UI:** server-rendered with no JavaScript. It refreshes itself while work is queued. +- **Security:** single password with login throttling, `SameSite` session cookie plus an Origin + check on every form post, strict Content-Security-Policy. Model output is rendered as Markdown + with raw HTML escaped. + +Run it locally against any Postgres: + +```bash +cp .env.example .env # set DATABASE_URL, CHAPTERWISE_BOOKS_DIR, secrets +chapterwise migrate +chapterwise serve & chapterwise worker +``` + +## Deployment + +Pushes to `main` run the tests, build the image to `ghcr.io/aminghuf/chapterwise:sha-`, then +deploy over SSH to the VPS. The deploy writes the image tag, backs up the database, runs +`docker compose up` (migrations run first) and waits for `/healthz`. Files are in [`deploy/`](deploy). + +**One-time setup:** + +1. **DNS:** `A read → `. +2. **Server:** copy `deploy/server-bootstrap.sh` to the VPS and run `bash server-bootstrap.sh`. It + creates `/root/chapterwise/.env` with generated secrets and asks for the web password and API key. +3. **Caddy:** add `deploy/read.aminghuf.com.caddy` to the Caddyfile and reload Caddy. +4. **GitHub secrets** (Settings → Secrets and variables → Actions): + - `DEPLOY_SSH_KEY`: a private key allowed to SSH in as root + - `DEPLOY_HOST`: the server's IP + - `DEPLOY_KNOWN_HOSTS`: the server's host-key line, from `ssh-keyscan -t ed25519 ` checked + against the server + +**On the server:** `cd /root/chapterwise && ./cw.sh ps | logs -f worker | backup`. + ## Configuration Set these in `.env` or the environment: @@ -63,18 +118,27 @@ Set these in `.env` or the environment: | `CHAPTERWISE_MODEL` | `deepseek-flash` | | | `CHAPTERWISE_CACHE_DIR` | `.cache/chapterwise` | | | `CHAPTERWISE_MAX_CHUNK_TOKENS` | `24000` | Approximate tokens per call | +| `DATABASE_URL` | | Service only; any `postgresql://` URL | +| `CHAPTERWISE_BOOKS_DIR` | `/books` | Folder scanned for books (mounted read-only in Docker) | +| `CHAPTERWISE_SECRET_KEY` | | Signs the login cookie | +| `CHAPTERWISE_ADMIN_PASSWORD` | | The web UI password | +| `CHAPTERWISE_PUBLIC_URL` | `http://localhost:8100` | With `https://`, cookies are marked Secure | +| `CHAPTERWISE_SCAN_INTERVAL` | `600` | Seconds between library scans | ## Development ```bash pytest # tests build their own EPUB/PDF/Markdown fixtures, no network or API key needed + +# Service tests need a Postgres they may wipe (skipped otherwise): +TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/chapterwise_test pytest ruff check . && ruff format . ``` ## Roadmap - [x] **Phase 1:** parsers, chunking, cached map-reduce summaries, CLI -- [ ] **Phase 2:** Postgres, background worker, books-folder sync, web UI at read.aminghuf.com, CI/CD to the VPS +- [x] **Phase 2:** Postgres, background worker, books-folder sync, web UI at read.aminghuf.com, CI/CD to the VPS - [ ] **Phase 3:** personal Telegram bot with scheduled delivery - [ ] **Phase 4:** whole-book summaries, micro-doses (original text plus recall questions), Markdown playbooks as a source diff --git a/deploy/cw.sh b/deploy/cw.sh new file mode 100755 index 0000000..c9a8ce8 --- /dev/null +++ b/deploy/cw.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# docker compose wrapper on the server, so .env and the pinned image are always used. +# ./cw.sh ps | ./cw.sh logs -f worker | ./cw.sh up -d | ./cw.sh backup [label] +set -euo pipefail +cd "$(dirname "$0")" + +compose() { + docker compose --env-file .env --env-file image.env -f docker-compose.yml "$@" +} + +if [ "${1:-}" = "backup" ]; then + # Database dump into ./backups, keeping the newest 7. Skipped when the database isn't running yet. + if ! compose ps --status running --services 2>/dev/null | grep -qx db; then + echo "Database not running; nothing to back up." + exit 0 + fi + mkdir -p backups + out="backups/chapterwise-$(date +%Y%m%d-%H%M%S)${2:+-$2}.sql.gz" + compose exec -T db pg_dump -U chapterwise -d chapterwise | gzip > "$out" + ls -1t backups/*.sql.gz | tail -n +8 | xargs -r rm -- + echo "Backup written: $out" + exit 0 +fi + +compose "$@" diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..18bb4ac --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,78 @@ +# chapterwise on the VPS. Run through ./cw.sh so .env and the pinned image (image.env) are applied. + +name: chapterwise + +x-app: &app + image: ${IMAGE:?set by image.env} + restart: unless-stopped + env_file: .env + environment: + DATABASE_URL: postgresql+psycopg://chapterwise:${POSTGRES_PASSWORD:?}@db:5432/chapterwise + CHAPTERWISE_BOOKS_DIR: /books + volumes: + # BookOrbit's library, read-only: chapterwise never changes your books. + - ${BOOKS_HOST_PATH:-/root/bookorbit/books}:/books:ro + read_only: true + tmpfs: + - /tmp + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +services: + db: + image: postgres:17-alpine + restart: unless-stopped + environment: + POSTGRES_USER: chapterwise + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + POSTGRES_DB: chapterwise + volumes: + - ./data/postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U chapterwise -d chapterwise"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 20s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + migrate: + <<: *app + command: ["chapterwise", "migrate"] + restart: "no" + healthcheck: + disable: true + depends_on: + db: + condition: service_healthy + + web: + <<: *app + command: ["chapterwise", "serve", "--host", "0.0.0.0", "--port", "8100"] + # Loopback only: Caddy on the host is the way in. + ports: + - "127.0.0.1:8100:8100" + depends_on: + migrate: + condition: service_completed_successfully + + worker: + <<: *app + command: ["chapterwise", "worker"] + healthcheck: + disable: true + stop_grace_period: 60s # let a running summary finish + depends_on: + migrate: + condition: service_completed_successfully diff --git a/deploy/read.aminghuf.com.caddy b/deploy/read.aminghuf.com.caddy new file mode 100644 index 0000000..afdec1b --- /dev/null +++ b/deploy/read.aminghuf.com.caddy @@ -0,0 +1,10 @@ +# Add to the VPS Caddyfile, then: systemctl reload caddy + +read.aminghuf.com { + encode zstd gzip + reverse_proxy 127.0.0.1:8100 + header { + Strict-Transport-Security "max-age=31536000" + -Server + } +} diff --git a/deploy/server-bootstrap.sh b/deploy/server-bootstrap.sh new file mode 100755 index 0000000..bdbe664 --- /dev/null +++ b/deploy/server-bootstrap.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# One-time setup on the VPS, as root: bash server-bootstrap.sh +# Creates /root/chapterwise with a .env holding generated secrets, and asks for the +# web password and the DeepSeek API key. Safe to re-run: existing values are kept. +set -euo pipefail + +DIR=/root/chapterwise +mkdir -p "$DIR"/{data/postgres,backups} +ENV="$DIR/.env" +touch "$ENV" +chmod 600 "$ENV" + +set_if_missing() { # name value + grep -q "^$1=" "$ENV" || echo "$1=$2" >> "$ENV" +} + +set_if_missing CHAPTERWISE_PUBLIC_URL https://read.aminghuf.com +set_if_missing BOOKS_HOST_PATH /root/bookorbit/books +set_if_missing CHAPTERWISE_MODEL deepseek-flash +set_if_missing TZ Europe/Rome +set_if_missing POSTGRES_PASSWORD "$(openssl rand -hex 24)" +set_if_missing CHAPTERWISE_SECRET_KEY "$(openssl rand -hex 32)" + +if ! grep -q '^CHAPTERWISE_ADMIN_PASSWORD=' "$ENV"; then + read -rsp "Password for the web UI: " pw; echo + [ -n "$pw" ] || { echo "Password can't be empty"; exit 1; } + echo "CHAPTERWISE_ADMIN_PASSWORD=$pw" >> "$ENV" +fi +if ! grep -q '^DEEPSEEK_API_KEY=' "$ENV"; then + read -rsp "DeepSeek API key (Enter to add later): " key; echo + echo "DEEPSEEK_API_KEY=$key" >> "$ENV" +fi + +[ -d /root/bookorbit/books ] || echo "Warning: /root/bookorbit/books doesn't exist; set BOOKS_HOST_PATH in $ENV" +echo "Ready: $ENV (edit it any time, then run ./cw.sh up -d)" diff --git a/pyproject.toml b/pyproject.toml index a48a13e..118c1c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "chapterwise" -version = "0.1.0" +version = "0.2.0" description = "Summarize your books chapter by chapter with an LLM and read them in micro-doses." readme = "README.md" license = "MIT" @@ -12,16 +12,26 @@ license-files = ["LICENSE"] requires-python = ">=3.12" authors = [{ name = "Seyedamin Ghazizadeh" }] dependencies = [ + "alembic>=1.13", "beautifulsoup4>=4.12", "ebooklib>=0.18", + "fastapi>=0.115", + "itsdangerous>=2.1", + "jinja2>=3.1", + "markdown-it-py>=3.0", "openai>=1.40", + "psycopg[binary]>=3.2", "pypdf>=5.0", "python-dotenv>=1.0", + "python-multipart>=0.0.9", + "sqlalchemy>=2.0", "typer>=0.12", + "uvicorn>=0.30", ] [project.optional-dependencies] dev = [ + "httpx>=0.27", # FastAPI test client "pytest>=8", "reportlab>=4", # builds PDF fixtures in tests "ruff>=0.6", diff --git a/src/chapterwise/__init__.py b/src/chapterwise/__init__.py index 09b78db..1e00ad4 100644 --- a/src/chapterwise/__init__.py +++ b/src/chapterwise/__init__.py @@ -1,3 +1,3 @@ """Summarize books chapter by chapter with an LLM.""" -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/src/chapterwise/cache.py b/src/chapterwise/cache.py index 87d537a..74b3102 100644 --- a/src/chapterwise/cache.py +++ b/src/chapterwise/cache.py @@ -1,6 +1,6 @@ """A content-addressed cache so the same request never hits the API twice. -Phase 1 stores JSON files on disk; the service will swap in a Postgres-backed cache with the same interface. +The CLI stores JSON files on disk; the service uses Postgres. Both share one interface. """ from __future__ import annotations @@ -13,6 +13,12 @@ from pathlib import Path from typing import Any, Protocol +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import sessionmaker + +from chapterwise.db import CacheEntry + class Cache(Protocol): def get(self, key: dict[str, Any]) -> dict[str, Any] | None: ... @@ -47,3 +53,29 @@ def set(self, key: dict[str, Any], value: dict[str, Any]) -> None: with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(record, f, ensure_ascii=False, indent=1) os.replace(tmp, path) + + +class PostgresCache: + """The same cache in the service's database (table llm_cache).""" + + def __init__(self, session_factory: sessionmaker): + self._sessions = session_factory + + def get(self, key: dict[str, Any]) -> dict[str, Any] | None: + with self._sessions() as session: + entry = session.get(CacheEntry, cache_digest(key)) + return entry.value if entry else None + + def set(self, key: dict[str, Any], value: dict[str, Any]) -> None: + stmt = insert(CacheEntry).values(digest=cache_digest(key), key=key, value=value) + stmt = stmt.on_conflict_do_update(index_elements=[CacheEntry.digest], set_={"value": value}) + with self._sessions() as session: + session.execute(stmt) + session.commit() + + def existing(self, digests: list[str]) -> set[str]: + """Which of these digests are cached (one query, for list pages).""" + if not digests: + return set() + with self._sessions() as session: + return set(session.scalars(select(CacheEntry.digest).where(CacheEntry.digest.in_(digests)))) diff --git a/src/chapterwise/cli.py b/src/chapterwise/cli.py index 327e84b..32dbc81 100644 --- a/src/chapterwise/cli.py +++ b/src/chapterwise/cli.py @@ -1,7 +1,8 @@ -"""Command line: inspect a book's sections and summarize them.""" +"""Command line: inspect and summarize books, and run the service (migrate, serve, worker).""" from __future__ import annotations +import logging from pathlib import Path from typing import Annotated @@ -152,6 +153,53 @@ def summarize( ) +# --- service ---------------------------------------------------------------------------------------------- + + +def _logging() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + + +@app.command() +def migrate() -> None: + """Create or upgrade the database schema.""" + from chapterwise.migrate import upgrade + + settings = Settings.from_env() + settings.require("database_url") + _logging() + upgrade(settings.database_url) + typer.echo("Database is up to date.") + + +@app.command() +def serve( + host: Annotated[str, typer.Option(help="Interface to listen on")] = "127.0.0.1", + port: Annotated[int, typer.Option(help="Port to listen on")] = 8100, +) -> None: + """Run the web UI.""" + import uvicorn + + from chapterwise.web import create_app + + _logging() + app_ = create_app(Settings.from_env()) + # Behind Caddy: trust X-Forwarded-* so logs and login throttling see the real client address. + uvicorn.run(app_, host=host, port=port, proxy_headers=True, forwarded_allow_ips="*") + + +@app.command() +def worker() -> None: + """Run the background worker: library scans, parsing and summaries.""" + from chapterwise.db import make_engine, make_session_factory + from chapterwise.worker import Worker + + settings = Settings.from_env() + settings.require("database_url") + _logging() + Worker(settings, make_session_factory(make_engine(settings.database_url))).run_forever() + + class _NoLLM: """Stand-in for --dry-run: knows the model name for cache keys, refuses to be called.""" diff --git a/src/chapterwise/config.py b/src/chapterwise/config.py index 7d2e827..4264f25 100644 --- a/src/chapterwise/config.py +++ b/src/chapterwise/config.py @@ -13,22 +13,67 @@ DEFAULT_MAX_CHUNK_TOKENS = 24_000 +def _int(name: str, default: int) -> int: + value = os.environ.get(name, "").strip() + return int(value) if value else default + + +def normalize_database_url(url: str | None) -> str | None: + """Accept plain postgres:// URLs and use the psycopg 3 driver for them.""" + if not url: + return None + for prefix in ("postgres://", "postgresql://"): + if url.startswith(prefix): + return "postgresql+psycopg://" + url[len(prefix) :] + return url + + @dataclass(frozen=True) class Settings: + # LLM api_key: str | None base_url: str model: str - cache_dir: Path max_chunk_tokens: int + # CLI cache (the service caches in Postgres instead) + cache_dir: Path + # Service + database_url: str | None = None + books_dir: Path = Path("/books") + secret_key: str | None = None + admin_password: str | None = None + public_url: str = "http://localhost:8100" + scan_interval: int = 600 # seconds between library scans + + @property + def secure_cookies(self) -> bool: + return self.public_url.startswith("https://") @classmethod def from_env(cls) -> Settings: load_dotenv() - max_chunk = os.environ.get("CHAPTERWISE_MAX_CHUNK_TOKENS", "").strip() + env = os.environ.get return cls( - api_key=os.environ.get("DEEPSEEK_API_KEY") or None, - base_url=os.environ.get("CHAPTERWISE_BASE_URL") or DEFAULT_BASE_URL, - model=os.environ.get("CHAPTERWISE_MODEL") or DEFAULT_MODEL, - cache_dir=Path(os.environ.get("CHAPTERWISE_CACHE_DIR") or ".cache/chapterwise"), - max_chunk_tokens=int(max_chunk) if max_chunk else DEFAULT_MAX_CHUNK_TOKENS, + api_key=env("DEEPSEEK_API_KEY") or None, + base_url=env("CHAPTERWISE_BASE_URL") or DEFAULT_BASE_URL, + model=env("CHAPTERWISE_MODEL") or DEFAULT_MODEL, + max_chunk_tokens=_int("CHAPTERWISE_MAX_CHUNK_TOKENS", DEFAULT_MAX_CHUNK_TOKENS), + cache_dir=Path(env("CHAPTERWISE_CACHE_DIR") or ".cache/chapterwise"), + database_url=normalize_database_url(env("DATABASE_URL")), + books_dir=Path(env("CHAPTERWISE_BOOKS_DIR") or "/books"), + secret_key=env("CHAPTERWISE_SECRET_KEY") or None, + admin_password=env("CHAPTERWISE_ADMIN_PASSWORD") or None, + public_url=(env("CHAPTERWISE_PUBLIC_URL") or "http://localhost:8100").rstrip("/"), + scan_interval=_int("CHAPTERWISE_SCAN_INTERVAL", 600), ) + + def require(self, *names: str) -> None: + """Fail early with a clear message when a service setting is missing.""" + env_names = { + "database_url": "DATABASE_URL", + "secret_key": "CHAPTERWISE_SECRET_KEY", + "admin_password": "CHAPTERWISE_ADMIN_PASSWORD", + } + missing = [env_names.get(n, n) for n in names if not getattr(self, n)] + if missing: + raise SystemExit(f"Missing required setting(s): {', '.join(missing)}") diff --git a/src/chapterwise/db.py b/src/chapterwise/db.py new file mode 100644 index 0000000..cb0e93f --- /dev/null +++ b/src/chapterwise/db.py @@ -0,0 +1,124 @@ +"""Postgres schema and session handling.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import ( + BigInteger, + Double, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, + create_engine, + func, + text, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.engine import Engine +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship, sessionmaker +from sqlalchemy.types import DateTime + +ACTIVE_JOB_STATUSES = ("queued", "running") + + +class Base(DeclarativeBase): + pass + + +class DocumentRow(Base): + """A book file found in the books folder.""" + + __tablename__ = "documents" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + path: Mapped[str] = mapped_column(Text, unique=True) # relative to the books folder + format: Mapped[str] = mapped_column(String(10)) + size: Mapped[int] = mapped_column(BigInteger) + mtime: Mapped[float] = mapped_column(Double) + sha256: Mapped[str | None] = mapped_column(String(64)) + title: Mapped[str] = mapped_column(Text) + author: Mapped[str | None] = mapped_column(Text) + language: Mapped[str | None] = mapped_column(String(35)) + # pending (waiting to be parsed) | ready | error | missing (file gone from the folder) + status: Mapped[str] = mapped_column(String(16), default="pending") + error: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + sections: Mapped[list[SectionRow]] = relationship( + back_populates="document", + order_by="SectionRow.position", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + +class SectionRow(Base): + __tablename__ = "sections" + __table_args__ = (UniqueConstraint("document_id", "position"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + document_id: Mapped[int] = mapped_column(ForeignKey("documents.id", ondelete="CASCADE"), index=True) + position: Mapped[int] = mapped_column(Integer) + title: Mapped[str] = mapped_column(Text) + level: Mapped[int] = mapped_column(Integer) + text: Mapped[str] = mapped_column(Text) + text_sha256: Mapped[str] = mapped_column(String(64)) + words: Mapped[int] = mapped_column(Integer) + tokens: Mapped[int] = mapped_column(Integer) + + document: Mapped[DocumentRow] = relationship(back_populates="sections") + + +class Job(Base): + """Background work: scan the library, parse a book, summarize a section.""" + + __tablename__ = "jobs" + __table_args__ = ( + # At most one queued/running job per dedupe key (e.g. the same section twice). + Index( + "jobs_active_dedupe", + "dedupe_key", + unique=True, + postgresql_where=text("status IN ('queued', 'running')"), + ), + Index("jobs_queue", "status", "run_after"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + kind: Mapped[str] = mapped_column(String(32)) + payload: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict) + dedupe_key: Mapped[str | None] = mapped_column(Text) + status: Mapped[str] = mapped_column(String(16), default="queued") # queued|running|done|failed + attempts: Mapped[int] = mapped_column(Integer, default=0) + error: Mapped[str | None] = mapped_column(Text) + run_after: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + +class CacheEntry(Base): + """LLM results, content-addressed. Summaries are read back from here for display.""" + + __tablename__ = "llm_cache" + + digest: Mapped[str] = mapped_column(String(64), primary_key=True) + key: Mapped[dict[str, Any]] = mapped_column(JSONB) + value: Mapped[dict[str, Any]] = mapped_column(JSONB) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + +def make_engine(database_url: str) -> Engine: + return create_engine(database_url, pool_pre_ping=True) + + +def make_session_factory(engine: Engine) -> sessionmaker: + return sessionmaker(engine, expire_on_commit=False) diff --git a/src/chapterwise/jobs.py b/src/chapterwise/jobs.py new file mode 100644 index 0000000..fbf9550 --- /dev/null +++ b/src/chapterwise/jobs.py @@ -0,0 +1,98 @@ +"""A small job queue in Postgres. Workers claim jobs with FOR UPDATE SKIP LOCKED, so no Redis is needed.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import func, select, update +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Session + +from chapterwise.db import ACTIVE_JOB_STATUSES, Job + +MAX_ATTEMPTS = 3 +RETRY_BASE_SECONDS = 30 + + +def enqueue( + session: Session, kind: str, payload: dict[str, Any] | None = None, dedupe_key: str | None = None +) -> bool: + """Queue a job. Returns False if an identical job (same dedupe_key) is already queued or running.""" + stmt = ( + insert(Job) + .values(kind=kind, payload=payload or {}, dedupe_key=dedupe_key, status="queued") + .on_conflict_do_nothing( + index_elements=[Job.dedupe_key], index_where=Job.status.in_(ACTIVE_JOB_STATUSES) + ) + .returning(Job.id) + ) + return session.execute(stmt).first() is not None + + +def claim(session: Session) -> Job | None: + """Take the oldest runnable job and mark it running. Safe with several workers.""" + next_id = ( + select(Job.id) + .where(Job.status == "queued", Job.run_after <= func.now()) + .order_by(Job.id) + .limit(1) + .with_for_update(skip_locked=True) + .scalar_subquery() + ) + stmt = ( + update(Job) + .where(Job.id == next_id) + .values(status="running", started_at=func.now(), attempts=Job.attempts + 1) + .returning(Job) + ) + job = session.scalars(stmt).first() + session.commit() + return job + + +def complete(session: Session, job_id: int) -> None: + session.execute( + update(Job).where(Job.id == job_id).values(status="done", error=None, finished_at=func.now()) + ) + session.commit() + + +def fail(session: Session, job_id: int, error: str, retry: bool = True) -> str: + """Record a failure. Retries with exponential backoff until MAX_ATTEMPTS. Returns the new status.""" + job = session.get(Job, job_id) + if job is None: + return "missing" + if retry and job.attempts < MAX_ATTEMPTS: + job.status = "queued" + job.run_after = datetime.now(UTC) + timedelta(seconds=RETRY_BASE_SECONDS * 2 ** (job.attempts - 1)) + else: + job.status = "failed" + job.finished_at = datetime.now(UTC) + job.error = error[:2000] + session.commit() + return job.status + + +def requeue_stale(session: Session) -> int: + """Jobs left 'running' by a worker that died (e.g. a restart) go back to the queue.""" + result = session.execute(update(Job).where(Job.status == "running").values(status="queued")) + session.commit() + return result.rowcount + + +def retry(session: Session, job_id: int) -> bool: + """Manually requeue a failed job (from the UI).""" + job = session.get(Job, job_id) + if job is None or job.status != "failed": + return False + active = session.scalar( + select(func.count()) + .select_from(Job) + .where(Job.dedupe_key == job.dedupe_key, Job.status.in_(ACTIVE_JOB_STATUSES)) + ) + if job.dedupe_key and active: + return False + job.status, job.attempts, job.error, job.run_after = "queued", 0, None, datetime.now(UTC) + session.commit() + return True diff --git a/src/chapterwise/library.py b/src/chapterwise/library.py new file mode 100644 index 0000000..8ab2fef --- /dev/null +++ b/src/chapterwise/library.py @@ -0,0 +1,142 @@ +"""The books folder: find files, keep the documents table in sync, parse books into sections.""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from pathlib import Path + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session + +from chapterwise import jobs +from chapterwise.db import DocumentRow, SectionRow +from chapterwise.models import Document, Section +from chapterwise.parsers import SUPPORTED_EXTENSIONS, ParseError, parse +from chapterwise.summarize import text_sha256 + +log = logging.getLogger(__name__) + +FORMATS = {".epub": "epub", ".pdf": "pdf", ".md": "md", ".markdown": "md"} + + +@dataclass +class ScanResult: + added: int = 0 + changed: int = 0 + missing: int = 0 + unchanged: int = 0 + + +def find_books(books_dir: Path) -> dict[str, tuple[int, float]]: + """Supported files under books_dir as {relative path: (size, mtime)}. Hidden folders are skipped.""" + if not books_dir.is_dir(): + raise FileNotFoundError(f"Books folder not found: {books_dir}") + found: dict[str, tuple[int, float]] = {} + for root, dirs, files in os.walk(books_dir): + dirs[:] = [d for d in dirs if not d.startswith(".")] + for name in files: + if name.startswith(".") or Path(name).suffix.lower() not in SUPPORTED_EXTENSIONS: + continue + path = Path(root, name) + try: + stat = path.stat() + except OSError: + continue + found[path.relative_to(books_dir).as_posix()] = (stat.st_size, stat.st_mtime) + return found + + +def guess_title(relative_path: str) -> str: + return Path(relative_path).stem.replace("_", " ").strip() or relative_path + + +def scan(session: Session, books_dir: Path) -> ScanResult: + """Register new and changed files (and queue them for parsing); mark vanished ones as missing.""" + found = find_books(books_dir) + existing = {doc.path: doc for doc in session.scalars(select(DocumentRow))} + result = ScanResult() + + for rel, (size, mtime) in found.items(): + doc = existing.get(rel) + if doc is None: + doc = DocumentRow( + path=rel, + format=FORMATS[Path(rel).suffix.lower()], + size=size, + mtime=mtime, + title=guess_title(rel), + status="pending", + ) + session.add(doc) + session.flush() + result.added += 1 + elif doc.size != size or doc.mtime != mtime or doc.status == "missing": + doc.size, doc.mtime, doc.status = size, mtime, "pending" + result.changed += 1 + else: + result.unchanged += 1 + continue + jobs.enqueue(session, "parse", {"document_id": doc.id}, dedupe_key=f"parse:{doc.id}") + + for rel, doc in existing.items(): + if rel not in found and doc.status != "missing": + doc.status = "missing" + result.missing += 1 + + session.commit() + return result + + +def parse_document(session: Session, books_dir: Path, document_id: int) -> None: + """Parse one file into sections (no LLM). A file that can't be parsed is marked 'error', not retried.""" + doc = session.get(DocumentRow, document_id) + if doc is None or doc.status == "missing": + return + try: + parsed = parse(books_dir / doc.path) + except ParseError as exc: + doc.status, doc.error = "error", str(exc) + session.commit() + log.warning("Can't parse %s: %s", doc.path, exc) + return + + session.execute(delete(SectionRow).where(SectionRow.document_id == doc.id)) + for section in parsed.sections: + session.add( + SectionRow( + document_id=doc.id, + position=section.index, + title=section.title, + level=section.level, + text=section.text, + text_sha256=text_sha256(section.text), + words=section.words, + tokens=section.tokens, + ) + ) + doc.sha256 = parsed.sha256 + doc.title = parsed.title + doc.author = parsed.author + doc.language = (parsed.language or "")[:35] or None + doc.status, doc.error = "ready", None + session.commit() + log.info("Parsed %s: %d sections", doc.path, len(parsed.sections)) + + +def to_models(doc: DocumentRow, section: SectionRow) -> tuple[Document, Section]: + """Bridge database rows to the parser's dataclasses, which the summarizer works with.""" + model_section = Section( + index=section.position, title=section.title, level=section.level, text=section.text + ) + model_doc = Document( + path=Path(doc.path), + format=doc.format, + title=doc.title, + author=doc.author, + language=doc.language, + sha256=doc.sha256 or "", + sections=(model_section,), + ) + return model_doc, model_section diff --git a/src/chapterwise/migrate.py b/src/chapterwise/migrate.py new file mode 100644 index 0000000..92489f5 --- /dev/null +++ b/src/chapterwise/migrate.py @@ -0,0 +1,21 @@ +"""Apply database migrations (Alembic), configured from code so no alembic.ini is needed.""" + +from __future__ import annotations + +from pathlib import Path + +from alembic import command +from alembic.config import Config + +MIGRATIONS_DIR = Path(__file__).parent / "migrations" + + +def alembic_config(database_url: str) -> Config: + config = Config() + config.set_main_option("script_location", str(MIGRATIONS_DIR)) + config.attributes["database_url"] = database_url + return config + + +def upgrade(database_url: str, revision: str = "head") -> None: + command.upgrade(alembic_config(database_url), revision) diff --git a/src/chapterwise/migrations/__init__.py b/src/chapterwise/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/chapterwise/migrations/env.py b/src/chapterwise/migrations/env.py new file mode 100644 index 0000000..7c67c9a --- /dev/null +++ b/src/chapterwise/migrations/env.py @@ -0,0 +1,22 @@ +"""Alembic environment. Run migrations with `chapterwise migrate`.""" + +from alembic import context +from sqlalchemy import create_engine + +from chapterwise.db import Base + +config = context.config +target_metadata = Base.metadata + + +def run_migrations_online() -> None: + # The URL is passed through attributes, not the ini file, so passwords with % don't break parsing. + engine = create_engine(config.attributes["database_url"]) + with engine.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata, compare_type=True) + with context.begin_transaction(): + context.run_migrations() + engine.dispose() + + +run_migrations_online() diff --git a/src/chapterwise/migrations/script.py.mako b/src/chapterwise/migrations/script.py.mako new file mode 100644 index 0000000..e67e889 --- /dev/null +++ b/src/chapterwise/migrations/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: str | None = ${repr(down_revision)} +branch_labels: str | Sequence[str] | None = ${repr(branch_labels)} +depends_on: str | Sequence[str] | None = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/src/chapterwise/migrations/versions/0001_initial_schema.py b/src/chapterwise/migrations/versions/0001_initial_schema.py new file mode 100644 index 0000000..8066b2f --- /dev/null +++ b/src/chapterwise/migrations/versions/0001_initial_schema.py @@ -0,0 +1,97 @@ +"""initial schema + +Revision ID: 0001 +Revises: +Create Date: 2026-11-01 00:16:29.587512 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0001" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "documents", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("format", sa.String(length=10), nullable=False), + sa.Column("size", sa.BigInteger(), nullable=False), + sa.Column("mtime", sa.Double(), nullable=False), + sa.Column("sha256", sa.String(length=64), nullable=True), + sa.Column("title", sa.Text(), nullable=False), + sa.Column("author", sa.Text(), nullable=True), + sa.Column("language", sa.String(length=35), nullable=True), + sa.Column("status", sa.String(length=16), nullable=False), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("path"), + ) + op.create_table( + "jobs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("dedupe_key", sa.Text(), nullable=True), + sa.Column("status", sa.String(length=16), nullable=False), + sa.Column("attempts", sa.Integer(), nullable=False), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("run_after", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "jobs_active_dedupe", + "jobs", + ["dedupe_key"], + unique=True, + postgresql_where=sa.text("status IN ('queued', 'running')"), + ) + op.create_index("jobs_queue", "jobs", ["status", "run_after"], unique=False) + op.create_table( + "llm_cache", + sa.Column("digest", sa.String(length=64), nullable=False), + sa.Column("key", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("value", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("digest"), + ) + op.create_table( + "sections", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("document_id", sa.Integer(), nullable=False), + sa.Column("position", sa.Integer(), nullable=False), + sa.Column("title", sa.Text(), nullable=False), + sa.Column("level", sa.Integer(), nullable=False), + sa.Column("text", sa.Text(), nullable=False), + sa.Column("text_sha256", sa.String(length=64), nullable=False), + sa.Column("words", sa.Integer(), nullable=False), + sa.Column("tokens", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["document_id"], ["documents.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("document_id", "position"), + ) + op.create_index(op.f("ix_sections_document_id"), "sections", ["document_id"], unique=False) + + +def downgrade() -> None: + op.drop_index(op.f("ix_sections_document_id"), table_name="sections") + op.drop_table("sections") + op.drop_table("llm_cache") + op.drop_index("jobs_queue", table_name="jobs") + op.drop_index( + "jobs_active_dedupe", table_name="jobs", postgresql_where=sa.text("status IN ('queued', 'running')") + ) + op.drop_table("jobs") + op.drop_table("documents") diff --git a/src/chapterwise/summarize.py b/src/chapterwise/summarize.py index 6966cd8..4ed3e21 100644 --- a/src/chapterwise/summarize.py +++ b/src/chapterwise/summarize.py @@ -37,10 +37,25 @@ class Estimate: input_tokens: int -def _sha(text: str) -> str: +def text_sha256(text: str) -> str: return hashlib.sha256(text.encode()).hexdigest() +def cache_key( + kind: str, text_hash: str, book_title: str, section_title: str, model: str, **extra: str +) -> dict[str, str]: + """The cache key of one LLM result. kind="section" is a finished section summary.""" + return { + "kind": kind, + "text_sha256": text_hash, + "book": book_title, + "section": section_title, + "model": model, + "prompt_version": prompts.PROMPT_VERSION, + **extra, + } + + class Summarizer: def __init__(self, llm: LLM, cache: Cache, max_chunk_tokens: int = 24_000): self.llm = llm @@ -118,15 +133,7 @@ def _condense(self, doc: Document, section: Section, notes: list[str], usage: Us return notes def _key(self, kind: str, text: str, doc: Document, section: Section, **extra: str) -> dict[str, str]: - return { - "kind": kind, - "text_sha256": _sha(text), - "book": doc.title, - "section": section.title, - "model": self.llm.model, - "prompt_version": prompts.PROMPT_VERSION, - **extra, - } + return cache_key(kind, text_sha256(text), doc.title, section.title, self.llm.model, **extra) def _cached_call( self, diff --git a/src/chapterwise/templates/base.html b/src/chapterwise/templates/base.html new file mode 100644 index 0000000..d2305f7 --- /dev/null +++ b/src/chapterwise/templates/base.html @@ -0,0 +1,81 @@ + + + + + +{% if refresh %}{% endif %} +{% block title %}chapterwise{% endblock %} + + + +
+
+ chapterwise + {% if request.session.get("user") %} + + {% endif %} +
+
+
+{% if request.session.get("user") and not api_key_set %} +

DEEPSEEK_API_KEY isn't set, so summarizing is turned off. Books can still be browsed.

+{% endif %} +{% block content %}{% endblock %} +
+ + diff --git a/src/chapterwise/templates/book.html b/src/chapterwise/templates/book.html new file mode 100644 index 0000000..edf8162 --- /dev/null +++ b/src/chapterwise/templates/book.html @@ -0,0 +1,56 @@ +{% extends "base.html" %} +{% block title %}{{ doc.title }} · chapterwise{% endblock %} +{% block content %} +

← Library

+

{{ doc.title }}

+

{% if doc.author %}{{ doc.author }} · {% endif %}{{ doc.format|upper }} · {{ doc.path }}

+ +{% if doc.status == "pending" %} +

Reading the table of contents… this page refreshes by itself.

+{% elif doc.status == "error" %} +

Can't read this file: {{ doc.error }}

+{% elif doc.status == "missing" %} +

This file is no longer in the books folder. Existing summaries are kept.

+{% endif %} + +{% if rows %} +
+ {% if missing_count %} +
+ + +
+ ≈ {{ missing_tokens|num }} tokens of text with {{ model }} + {% else %} + Every section is summarized or queued. + {% endif %} +
+ +
+ + + + {% for row in rows %} + {% set s = row.section %} + + + + + + + {% endfor %} + +
#SectionWordsSummary
{{ s.position }}{{ s.title }}{{ s.words|num }} + {% if row.state == "done" %}Read + {% elif row.state == "running" %}working… + {% elif row.state == "queued" %}queued + {% else %} +
+ + +
+ {% endif %} +
+
+{% endif %} +{% endblock %} diff --git a/src/chapterwise/templates/jobs.html b/src/chapterwise/templates/jobs.html new file mode 100644 index 0000000..66ba6ab --- /dev/null +++ b/src/chapterwise/templates/jobs.html @@ -0,0 +1,39 @@ +{% extends "base.html" %} +{% block title %}Jobs · chapterwise{% endblock %} +{% block content %} +

Jobs

+

+ {% for status in ["queued", "running", "done", "failed"] %}{{ status }}: {{ queue.get(status, 0)|num }}{% if not loop.last %} · {% endif %}{% endfor %} +

+{% if not jobs %} +

No jobs yet.

+{% else %} +
+ + + + {% for job in jobs %} + {% set doc_id = job.payload.get("document_id") %} + + + + + + + {% endfor %} + +
#JobStatusWhen
{{ job.id }} + {{ job.kind }} + {% if doc_id %}· {{ titles.get(doc_id, "book " ~ doc_id) }}{% endif %} + {% if job.payload.get("position") is not none %}· section {{ job.payload["position"] }}{% endif %} + {% if job.error %}
{{ job.error[:300] }}
{% endif %} + 		+ {{ job.status }} + {% if job.attempts > 1 %}×{{ job.attempts }}{% endif %} + {% if job.status == "failed" %} +
+ {% endif %} + 		{{ (job.finished_at or job.started_at or job.created_at).strftime("%d %b %H:%M") }}
+
+{% endif %} +{% endblock %} diff --git a/src/chapterwise/templates/library.html b/src/chapterwise/templates/library.html new file mode 100644 index 0000000..c21c58e --- /dev/null +++ b/src/chapterwise/templates/library.html @@ -0,0 +1,42 @@ +{% extends "base.html" %} +{% block title %}Library · chapterwise{% endblock %} +{% block content %} +

Library

+

+ {{ docs|length }} book{{ "" if docs|length == 1 else "s" }}{% if q %} matching “{{ q }}”{% endif %} + {% if queue.get("queued") or queue.get("running") %} · {{ (queue.get("queued", 0) + queue.get("running", 0))|num }} job(s) in progress{% endif %} + {% if queue.get("failed") %} · {{ queue["failed"] }} failed{% endif %} +

+ +
+
+
+
+ +{% if not docs %} +

{% if q %}No books match.{% else %}No books yet. Add EPUB, PDF or Markdown files to the books folder, then rescan.{% endif %}

+{% else %} +
+ + + + {% for doc in docs %} + {% set s = stats.get(doc.id, {"sections": 0, "summarized": 0}) %} + + + + + + + {% endfor %} + +
BookFormatSectionsSummarized
+ {{ doc.title }} + {% if doc.author %}
{{ doc.author }}
{% endif %} + {% if doc.status == "pending" %}preparing…{% endif %} + {% if doc.status == "error" %}can't read{% endif %} + {% if doc.status == "missing" %}file removed{% endif %} + 		{{ doc.format|upper }}{{ s.sections|num }}{% if s.sections %}{{ s.summarized }}/{{ s.sections }}{% else %}–{% endif %}
+
+{% endif %} +{% endblock %} diff --git a/src/chapterwise/templates/login.html b/src/chapterwise/templates/login.html new file mode 100644 index 0000000..0e72bc0 --- /dev/null +++ b/src/chapterwise/templates/login.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} +{% block title %}Log in · chapterwise{% endblock %} +{% block content %} +
+

Log in

+ {% if error %}

{{ error }}

{% endif %} +
+ +

+ +
+
+{% endblock %} diff --git a/src/chapterwise/templates/section.html b/src/chapterwise/templates/section.html new file mode 100644 index 0000000..255f125 --- /dev/null +++ b/src/chapterwise/templates/section.html @@ -0,0 +1,32 @@ +{% extends "base.html" %} +{% block title %}{{ section.title }} · {{ doc.title }}{% endblock %} +{% block content %} +

← {{ doc.title }}

+

{{ section.title }}

+

{{ section.words|num }} words in the original

+ +
+{% if summary %} + {{ summary|markdown|safe }} +{% elif state == "running" %} +

Summarizing now… this page refreshes by itself.

+{% elif state == "queued" %} +

Queued for summarizing… this page refreshes by itself.

+{% else %} +
+ + +
+{% endif %} +
+ +
+ Original text + 
{{ section.text }}
+
+ + +{% endblock %} diff --git a/src/chapterwise/web.py b/src/chapterwise/web.py new file mode 100644 index 0000000..7cfdf09 --- /dev/null +++ b/src/chapterwise/web.py @@ -0,0 +1,339 @@ +"""The web UI: a single-user, server-rendered app (no JavaScript).""" +# No `from __future__ import annotations` here: FastAPI must see the real types of the local DB/User aliases. + +import hmac +import time +from collections import defaultdict, deque +from pathlib import Path +from typing import Annotated +from urllib.parse import quote, urlparse + +from fastapi import Depends, FastAPI, Form, HTTPException, Request +from fastapi.responses import HTMLResponse, PlainTextResponse, RedirectResponse, Response +from fastapi.templating import Jinja2Templates +from markdown_it import MarkdownIt +from sqlalchemy import func, select, text +from sqlalchemy.orm import Session, sessionmaker +from starlette.middleware.sessions import SessionMiddleware + +from chapterwise import jobs +from chapterwise.cache import PostgresCache, cache_digest +from chapterwise.config import Settings +from chapterwise.db import ( + ACTIVE_JOB_STATUSES, + DocumentRow, + Job, + SectionRow, + make_engine, + make_session_factory, +) +from chapterwise.summarize import cache_key + +TEMPLATES_DIR = Path(__file__).parent / "templates" +LOGIN_MAX_FAILURES = 10 +LOGIN_WINDOW_SECONDS = 15 * 60 +SESSION_MAX_AGE = 30 * 24 * 3600 + +# Raw HTML in model output is escaped, never rendered. +_markdown = MarkdownIt("commonmark", {"html": False}).enable(["table", "strikethrough"]) + +SECURITY_HEADERS = { + "Content-Security-Policy": ( + "default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; " + "script-src 'none'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'" + ), + "X-Content-Type-Options": "nosniff", + "Referrer-Policy": "same-origin", + "X-Frame-Options": "DENY", +} + + +class LoginRequired(Exception): + pass + + +class LoginThrottle: + """Too many wrong passwords from one address locks it out for a while.""" + + def __init__(self) -> None: + self._failures: dict[str, deque[float]] = defaultdict(deque) + + def blocked(self, ip: str) -> bool: + failures = self._failures[ip] + while failures and failures[0] < time.monotonic() - LOGIN_WINDOW_SECONDS: + failures.popleft() + return len(failures) >= LOGIN_MAX_FAILURES + + def record_failure(self, ip: str) -> None: + self._failures[ip].append(time.monotonic()) + + def reset(self, ip: str) -> None: + self._failures.pop(ip, None) + + +def create_app(settings: Settings, sessions: sessionmaker | None = None) -> FastAPI: + settings.require("database_url", "secret_key", "admin_password") + if sessions is None: + sessions = make_session_factory(make_engine(settings.database_url)) + cache = PostgresCache(sessions) + throttle = LoginThrottle() + templates = Jinja2Templates(directory=str(TEMPLATES_DIR)) + templates.env.filters["num"] = lambda n: f"{n or 0:,}" + templates.env.filters["markdown"] = _markdown.render + + app = FastAPI(title="chapterwise", docs_url=None, redoc_url=None, openapi_url=None) + app.add_middleware( + SessionMiddleware, + secret_key=settings.secret_key, + session_cookie="chapterwise_session", + max_age=SESSION_MAX_AGE, + same_site="lax", + https_only=settings.secure_cookies, + ) + + @app.middleware("http") + async def protect(request: Request, call_next): + # Reject cross-site form posts (on top of SameSite cookies). + if request.method not in ("GET", "HEAD", "OPTIONS"): + source = request.headers.get("origin") or request.headers.get("referer") + if source and urlparse(source).netloc != request.headers.get("host"): + return PlainTextResponse("Cross-site request blocked", status_code=403) + response = await call_next(request) + for name, value in SECURITY_HEADERS.items(): + response.headers.setdefault(name, value) + return response + + @app.exception_handler(LoginRequired) + async def to_login(request: Request, _exc: LoginRequired): + target = request.url.path + (f"?{request.url.query}" if request.url.query else "") + return RedirectResponse(f"/login?next={quote(target)}", status_code=303) + + def db(): + with sessions() as session: + yield session + + def user(request: Request) -> str: + if request.session.get("user") != "admin": + raise LoginRequired + return "admin" + + DB = Annotated[Session, Depends(db)] + User = Annotated[str, Depends(user)] + + def render(request: Request, name: str, **context) -> HTMLResponse: + return templates.TemplateResponse( + request, + name, + {"api_key_set": bool(settings.api_key), "model": settings.model, **context}, + ) + + def summary_digest(doc: DocumentRow, section: SectionRow) -> str: + return cache_digest( + cache_key("section", section.text_sha256, doc.title, section.title, settings.model) + ) + + def queue_counts(session: Session) -> dict[str, int]: + rows = session.execute(select(Job.status, func.count()).group_by(Job.status)).all() + return {status: count for status, count in rows} + + def active_summaries(session: Session, document_id: int) -> dict[int, str]: + rows = session.execute( + select(Job.payload, Job.status).where( + Job.kind == "summarize", + Job.status.in_(ACTIVE_JOB_STATUSES), + Job.dedupe_key.like(f"summarize:{document_id}:%"), + ) + ).all() + return {payload["position"]: status for payload, status in rows} + + # --- health ------------------------------------------------------------------------------------------- + + @app.get("/healthz", include_in_schema=False) + def healthz(session: DB) -> dict[str, str]: + session.execute(text("SELECT 1")) + return {"status": "ok"} + + # --- auth --------------------------------------------------------------------------------------------- + + @app.get("/login", response_class=HTMLResponse) + def login_form(request: Request, next: str = "/"): + return render(request, "login.html", next=next, error=None) + + @app.post("/login") + def login(request: Request, password: Annotated[str, Form()], next: Annotated[str, Form()] = "/"): + ip = request.client.host if request.client else "unknown" + if throttle.blocked(ip): + return render(request, "login.html", next=next, error="Too many attempts. Try again later.") + if not hmac.compare_digest(password.encode(), settings.admin_password.encode()): + throttle.record_failure(ip) + return render(request, "login.html", next=next, error="Wrong password.") + throttle.reset(ip) + request.session.clear() + request.session["user"] = "admin" + safe_next = next if next.startswith("/") and not next.startswith("//") else "/" + return RedirectResponse(safe_next, status_code=303) + + @app.post("/logout") + def logout(request: Request): + request.session.clear() + return RedirectResponse("/login", status_code=303) + + # --- library ------------------------------------------------------------------------------------------ + + @app.get("/", response_class=HTMLResponse) + def library(request: Request, session: DB, _user: User, q: str = ""): + query = select(DocumentRow).order_by(func.lower(DocumentRow.title)) + if q.strip(): + pattern = f"%{q.strip()}%" + query = query.where(DocumentRow.title.ilike(pattern) | DocumentRow.author.ilike(pattern)) + docs = session.scalars(query).all() + + sections = session.execute( + select(SectionRow.document_id, SectionRow.title, SectionRow.text_sha256).where( + SectionRow.document_id.in_([d.id for d in docs]) + ) + ).all() + by_id = {d.id: d for d in docs} + digests: dict[int, list[str]] = defaultdict(list) + for document_id, title, sha in sections: + doc_title = by_id[document_id].title + digests[document_id].append( + cache_digest(cache_key("section", sha, doc_title, title, settings.model)) + ) + done = cache.existing([d for ds in digests.values() for d in ds]) + stats = { + doc_id: {"sections": len(ds), "summarized": sum(1 for d in ds if d in done)} + for doc_id, ds in digests.items() + } + counts = queue_counts(session) + return render( + request, + "library.html", + docs=docs, + stats=stats, + q=q, + queue=counts, + refresh=bool(counts.get("queued") or counts.get("running")), + ) + + @app.post("/rescan") + def rescan(session: DB, _user: User): + jobs.enqueue(session, "scan", dedupe_key="scan") + session.commit() + return RedirectResponse("/", status_code=303) + + # --- books -------------------------------------------------------------------------------------------- + + def get_doc(session: Session, document_id: int) -> DocumentRow: + doc = session.get(DocumentRow, document_id) + if doc is None: + raise HTTPException(404, "Book not found") + return doc + + @app.get("/b/{document_id}", response_class=HTMLResponse) + def book(request: Request, document_id: int, session: DB, _user: User): + doc = get_doc(session, document_id) + sections = doc.sections + done = cache.existing([summary_digest(doc, s) for s in sections]) + active = active_summaries(session, doc.id) + rows = [ + { + "section": s, + "state": "done" if summary_digest(doc, s) in done else active.get(s.position, "none"), + } + for s in sections + ] + missing = [r["section"] for r in rows if r["state"] == "none"] + top_level = min((s.level for s in sections), default=1) + return render( + request, + "book.html", + doc=doc, + rows=rows, + top_level=top_level, + missing_count=len(missing), + missing_tokens=sum(s.tokens for s in missing), + refresh=doc.status == "pending" or bool(active), + ) + + @app.post("/b/{document_id}/summarize") + def summarize(document_id: int, session: DB, _user: User, sections: Annotated[str, Form()]): + if not settings.api_key: + raise HTTPException(400, "DEEPSEEK_API_KEY is not set") + doc = get_doc(session, document_id) + if sections == "all": + done = cache.existing([summary_digest(doc, s) for s in doc.sections]) + positions = [s.position for s in doc.sections if summary_digest(doc, s) not in done] + else: + try: + positions = [int(sections)] + except ValueError: + raise HTTPException(400, "Invalid section") from None + if positions[0] not in {s.position for s in doc.sections}: + raise HTTPException(404, "Section not found") + for position in positions: + jobs.enqueue( + session, + "summarize", + {"document_id": doc.id, "position": position}, + dedupe_key=f"summarize:{doc.id}:{position}", + ) + session.commit() + anchor = f"#s{positions[0]}" if len(positions) == 1 else "" + return RedirectResponse(f"/b/{doc.id}{anchor}", status_code=303) + + @app.get("/b/{document_id}/s/{position}", response_class=HTMLResponse) + def section_page(request: Request, document_id: int, position: int, session: DB, _user: User): + doc = get_doc(session, document_id) + by_position = {s.position: s for s in doc.sections} + section = by_position.get(position) + if section is None: + raise HTTPException(404, "Section not found") + entry = cache.get(cache_key("section", section.text_sha256, doc.title, section.title, settings.model)) + state = "done" if entry else active_summaries(session, doc.id).get(position, "none") + positions = sorted(by_position) + index = positions.index(position) + return render( + request, + "section.html", + doc=doc, + section=section, + summary=entry["text"] if entry else None, + state=state, + prev=by_position[positions[index - 1]] if index > 0 else None, + next=by_position[positions[index + 1]] if index + 1 < len(positions) else None, + refresh=state in ACTIVE_JOB_STATUSES, + ) + + # --- jobs --------------------------------------------------------------------------------------------- + + @app.get("/jobs", response_class=HTMLResponse) + def job_list(request: Request, session: DB, _user: User): + recent = session.scalars(select(Job).order_by(Job.id.desc()).limit(100)).all() + titles = dict( + session.execute( + select(DocumentRow.id, DocumentRow.title).where( + DocumentRow.id.in_({j.payload.get("document_id") for j in recent} - {None}) + ) + ).all() + ) + counts = queue_counts(session) + return render( + request, + "jobs.html", + jobs=recent, + titles=titles, + queue=counts, + refresh=bool(counts.get("queued") or counts.get("running")), + ) + + @app.post("/jobs/{job_id}/retry") + def retry_job(job_id: int, session: DB, _user: User): + jobs.retry(session, job_id) + return RedirectResponse("/jobs", status_code=303) + + @app.get("/favicon.ico", include_in_schema=False) + def favicon(): + return Response(status_code=204) + + return app diff --git a/src/chapterwise/worker.py b/src/chapterwise/worker.py new file mode 100644 index 0000000..f52d06a --- /dev/null +++ b/src/chapterwise/worker.py @@ -0,0 +1,125 @@ +"""The background worker: scans the library on a timer and runs queued jobs one at a time.""" + +from __future__ import annotations + +import logging +import signal +import threading +import time + +from sqlalchemy import select +from sqlalchemy.orm import sessionmaker + +from chapterwise import jobs +from chapterwise.cache import PostgresCache +from chapterwise.config import Settings +from chapterwise.db import DocumentRow, Job, SectionRow +from chapterwise.library import parse_document, scan, to_models +from chapterwise.llm import LLM, OpenAICompatibleLLM +from chapterwise.summarize import Summarizer + +log = logging.getLogger(__name__) + +POLL_SECONDS = 2.0 + + +class PermanentError(Exception): + """A job that can't succeed by retrying (missing book, no API key...).""" + + +class Worker: + def __init__(self, settings: Settings, sessions: sessionmaker, llm: LLM | None = None): + self.settings = settings + self.sessions = sessions + if llm is None and settings.api_key: + llm = OpenAICompatibleLLM(settings.api_key, settings.base_url, settings.model) + self.summarizer = ( + Summarizer(llm, PostgresCache(sessions), settings.max_chunk_tokens) if llm is not None else None + ) + self._stop = threading.Event() + self._last_scan = float("-inf") + + def stop(self, *_args) -> None: + log.info("Stopping after the current job") + self._stop.set() + + def run_forever(self) -> None: + signal.signal(signal.SIGTERM, self.stop) + signal.signal(signal.SIGINT, self.stop) + with self.sessions() as session: + if requeued := jobs.requeue_stale(session): + log.info("Requeued %d job(s) interrupted by the last shutdown", requeued) + log.info("Worker started (books: %s, model: %s)", self.settings.books_dir, self.settings.model) + while not self._stop.is_set(): + self.schedule_scan() + if not self.run_once(): + self._stop.wait(POLL_SECONDS) + + def schedule_scan(self) -> None: + now = time.monotonic() + if now - self._last_scan >= self.settings.scan_interval: + with self.sessions() as session: + jobs.enqueue(session, "scan", dedupe_key="scan") + session.commit() + self._last_scan = now + + def run_once(self) -> bool: + """Run one job if there is one. Returns False when the queue is empty.""" + with self.sessions() as session: + job = jobs.claim(session) + if job is None: + return False + + started = time.monotonic() + try: + self.handle(job) + except PermanentError as exc: + with self.sessions() as session: + jobs.fail(session, job.id, str(exc), retry=False) + log.error("Job %d (%s) failed: %s", job.id, job.kind, exc) + except Exception as exc: + with self.sessions() as session: + status = jobs.fail(session, job.id, f"{type(exc).__name__}: {exc}") + log.exception("Job %d (%s) failed, now %s", job.id, job.kind, status) + else: + with self.sessions() as session: + jobs.complete(session, job.id) + log.info("Job %d (%s) done in %.1fs", job.id, job.kind, time.monotonic() - started) + return True + + def handle(self, job: Job) -> None: + if job.kind == "scan": + with self.sessions() as session: + result = scan(session, self.settings.books_dir) + log.info("Scan: %s", result) + elif job.kind == "parse": + with self.sessions() as session: + parse_document(session, self.settings.books_dir, job.payload["document_id"]) + elif job.kind == "summarize": + self.summarize(job.payload["document_id"], job.payload["position"]) + else: + raise PermanentError(f"Unknown job kind '{job.kind}'") + + def summarize(self, document_id: int, position: int) -> None: + if self.summarizer is None: + raise PermanentError("DEEPSEEK_API_KEY is not set") + with self.sessions() as session: + doc = session.get(DocumentRow, document_id) + section = session.scalar( + select(SectionRow).where( + SectionRow.document_id == document_id, SectionRow.position == position + ) + ) + if doc is None or section is None: + raise PermanentError(f"Section {position} of document {document_id} no longer exists") + model_doc, model_section = to_models(doc, section) + _, usage = self.summarizer.summarize_section(model_doc, model_section) + log.info( + "Summarized '%s' / '%s': %d call(s), %d cached, %d tokens in, %d out", + model_doc.title, + model_section.title, + usage.llm_calls, + usage.cache_hits, + usage.tokens_in, + usage.tokens_out, + ) diff --git a/tests/conftest.py b/tests/conftest.py index ee5e6e9..f030d21 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -126,3 +126,82 @@ def complete(self, system: str, user: str) -> Completion: @pytest.fixture def fake_llm() -> FakeLLM: return FakeLLM() + + +# --- Postgres (service tests) ----------------------------------------------------------------------------- +# Set TEST_DATABASE_URL to a database the tests may wipe, e.g. +# postgresql+psycopg://postgres:postgres@localhost:5432/chapterwise_test +# Tests that need it are skipped when it isn't set. + + +@pytest.fixture(scope="session") +def database_url() -> str: + import os + + from chapterwise.config import normalize_database_url + + url = normalize_database_url(os.environ.get("TEST_DATABASE_URL")) + if not url: + pytest.skip("TEST_DATABASE_URL is not set") + return url + + +@pytest.fixture(scope="session") +def migrated_engine(database_url): + from sqlalchemy import text + + from chapterwise.db import make_engine + from chapterwise.migrate import upgrade + + engine = make_engine(database_url) + with engine.begin() as conn: # start from an empty schema so the migration itself is tested + conn.execute(text("DROP SCHEMA public CASCADE; CREATE SCHEMA public")) + upgrade(database_url) + yield engine + engine.dispose() + + +@pytest.fixture +def sessions(migrated_engine): + from sqlalchemy import text + + from chapterwise.db import make_session_factory + + with migrated_engine.begin() as conn: + conn.execute(text("TRUNCATE documents, sections, jobs, llm_cache RESTART IDENTITY CASCADE")) + return make_session_factory(migrated_engine) + + +@pytest.fixture +def books_dir(tmp_path: Path, sample_epub: Path) -> Path: + """A books folder laid out like BookOrbit's: Author/Title/file.""" + folder = tmp_path / "books" + (folder / "Ada Tester" / "The Test Book").mkdir(parents=True) + sample_epub.rename(folder / "Ada Tester" / "The Test Book" / "test-book.epub") + (folder / "notes").mkdir() + (folder / "notes" / "playbook.md").write_text( + f"# Playbook\n\n## One\n\n{words('one')}\n\n## Two\n\n{words('two')}\n", encoding="utf-8" + ) + (folder / "notes" / "ignored.txt").write_text("not a book") + (folder / ".hidden").mkdir() + (folder / ".hidden" / "secret.md").write_text(words("hidden")) + return folder + + +@pytest.fixture +def settings(database_url, books_dir): + from chapterwise.config import Settings + + return Settings( + api_key="sk-test", + base_url="http://127.0.0.1:9", + model="fake-model", + max_chunk_tokens=24_000, + cache_dir=Path("/nonexistent"), + database_url=database_url, + books_dir=books_dir, + secret_key="test-secret", + admin_password="correct horse", + public_url="http://testserver", + scan_interval=600, + ) diff --git a/tests/test_jobs.py b/tests/test_jobs.py new file mode 100644 index 0000000..ea2476e --- /dev/null +++ b/tests/test_jobs.py @@ -0,0 +1,89 @@ +from datetime import UTC, datetime, timedelta + +from chapterwise import jobs +from chapterwise.db import Job + + +def test_enqueue_dedupes_active_jobs(sessions): + with sessions() as s: + assert jobs.enqueue(s, "summarize", {"position": 1}, dedupe_key="summarize:1:1") + assert not jobs.enqueue(s, "summarize", {"position": 1}, dedupe_key="summarize:1:1") + assert jobs.enqueue(s, "summarize", {"position": 2}, dedupe_key="summarize:1:2") + assert jobs.enqueue(s, "scan") # no dedupe key: always queued + assert jobs.enqueue(s, "scan") + s.commit() + assert s.query(Job).count() == 4 + + +def test_finished_job_can_be_queued_again(sessions): + with sessions() as s: + jobs.enqueue(s, "scan", dedupe_key="scan") + s.commit() + job = jobs.claim(s) + jobs.complete(s, job.id) + assert jobs.enqueue(s, "scan", dedupe_key="scan") + + +def test_claim_takes_oldest_and_two_workers_never_share(sessions): + with sessions() as s: + for n in range(3): + jobs.enqueue(s, "scan", {"n": n}) + s.commit() + + with sessions() as a, sessions() as b: + first = jobs.claim(a) + second = jobs.claim(b) + assert first.payload == {"n": 0} and first.status == "running" and first.attempts == 1 + assert second.payload == {"n": 1} + with sessions() as s: + assert jobs.claim(s).payload == {"n": 2} + assert jobs.claim(s) is None + + +def test_skip_locked(sessions): + with sessions() as s: + jobs.enqueue(s, "scan", {"n": 0}) + jobs.enqueue(s, "scan", {"n": 1}) + s.commit() + # A transaction holding a row lock (as a worker mid-claim would) doesn't block the next claim. + with sessions() as holder, sessions() as other: + holder.execute(Job.__table__.select().where(Job.id == 1).with_for_update()) + assert jobs.claim(other).payload == {"n": 1} + holder.rollback() + + +def test_failure_retries_with_backoff_then_gives_up(sessions): + with sessions() as s: + jobs.enqueue(s, "summarize", dedupe_key="k") + s.commit() + for attempt in range(1, jobs.MAX_ATTEMPTS + 1): + s.execute(Job.__table__.update().values(run_after=datetime.now(UTC) - timedelta(seconds=1))) + s.commit() + job = jobs.claim(s) + assert job.attempts == attempt + status = jobs.fail(s, job.id, "boom") + assert status == ("failed" if attempt == jobs.MAX_ATTEMPTS else "queued") + if status == "queued": + assert s.get(Job, job.id).run_after > datetime.now(UTC) # backoff + assert s.get(Job, job.id).error == "boom" + + +def test_permanent_failure_and_manual_retry(sessions): + with sessions() as s: + jobs.enqueue(s, "summarize", dedupe_key="k") + s.commit() + job = jobs.claim(s) + assert jobs.fail(s, job.id, "no key", retry=False) == "failed" + assert jobs.retry(s, job.id) + refreshed = s.get(Job, job.id) + assert (refreshed.status, refreshed.attempts, refreshed.error) == ("queued", 0, None) + assert not jobs.retry(s, job.id) # only failed jobs + + +def test_requeue_stale(sessions): + with sessions() as s: + jobs.enqueue(s, "scan") + s.commit() + jobs.claim(s) + assert jobs.requeue_stale(s) == 1 + assert jobs.claim(s) is not None diff --git a/tests/test_library.py b/tests/test_library.py new file mode 100644 index 0000000..088804e --- /dev/null +++ b/tests/test_library.py @@ -0,0 +1,67 @@ +import os + +from sqlalchemy import select + +from chapterwise.db import DocumentRow, Job, SectionRow +from chapterwise.library import find_books, parse_document, scan + + +def test_find_books_skips_hidden_and_unsupported(books_dir): + assert sorted(find_books(books_dir)) == ["Ada Tester/The Test Book/test-book.epub", "notes/playbook.md"] + + +def test_scan_registers_new_files_and_queues_parsing(sessions, books_dir): + with sessions() as s: + result = scan(s, books_dir) + assert (result.added, result.changed, result.missing) == (2, 0, 0) + docs = s.scalars(select(DocumentRow).order_by(DocumentRow.path)).all() + assert [d.status for d in docs] == ["pending", "pending"] + assert docs[0].format == "epub" and docs[1].format == "md" + assert sorted(j.dedupe_key for j in s.scalars(select(Job))) == ["parse:1", "parse:2"] + + again = scan(s, books_dir) + assert (again.added, again.changed, again.unchanged) == (0, 0, 2) + + +def test_scan_detects_changes_and_removals(sessions, books_dir): + with sessions() as s: + scan(s, books_dir) + playbook = books_dir / "notes" / "playbook.md" + playbook.write_text(playbook.read_text() + "\nmore text\n") + os.utime(playbook, (1_000_000_000, 1_000_000_000)) + (books_dir / "Ada Tester" / "The Test Book" / "test-book.epub").unlink() + + result = scan(s, books_dir) + assert (result.changed, result.missing) == (1, 1) + statuses = {d.path: d.status for d in s.scalars(select(DocumentRow))} + assert statuses == { + "notes/playbook.md": "pending", + "Ada Tester/The Test Book/test-book.epub": "missing", + } + + +def test_parse_document_stores_sections(sessions, books_dir): + with sessions() as s: + scan(s, books_dir) + epub = s.scalar(select(DocumentRow).where(DocumentRow.format == "epub")) + parse_document(s, books_dir, epub.id) + + s.refresh(epub) + assert (epub.status, epub.title, epub.author) == ("ready", "The Test Book", "Ada Tester") + assert len(epub.sha256) == 64 + sections = s.scalars(select(SectionRow).order_by(SectionRow.position)).all() + assert [x.title for x in sections] == ["Chapter 1", "Chapter 2", "Chapter 3"] + assert all(len(x.text_sha256) == 64 and x.words >= 20 for x in sections) + + parse_document(s, books_dir, epub.id) # re-parsing replaces, never duplicates + assert len(s.scalars(select(SectionRow)).all()) == 3 + + +def test_unreadable_file_is_marked_error(sessions, books_dir): + (books_dir / "broken.pdf").write_bytes(b"not really a pdf") + with sessions() as s: + scan(s, books_dir) + broken = s.scalar(select(DocumentRow).where(DocumentRow.path == "broken.pdf")) + parse_document(s, books_dir, broken.id) + s.refresh(broken) + assert broken.status == "error" and broken.error diff --git a/tests/test_web.py b/tests/test_web.py new file mode 100644 index 0000000..a911511 --- /dev/null +++ b/tests/test_web.py @@ -0,0 +1,134 @@ +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import select + +from chapterwise.cache import PostgresCache +from chapterwise.db import DocumentRow, Job, SectionRow +from chapterwise.summarize import cache_key +from chapterwise.web import create_app +from chapterwise.worker import Worker +from tests.conftest import FakeLLM + + +@pytest.fixture +def library(sessions, settings): + """A scanned and parsed library.""" + worker = Worker(settings, sessions, llm=FakeLLM("fake-model")) + worker.schedule_scan() + while worker.run_once(): + pass + return worker + + +@pytest.fixture +def client(sessions, settings): + return TestClient(create_app(settings, sessions)) + + +@pytest.fixture +def logged_in(client): + response = client.post("/login", data={"password": "correct horse", "next": "/"}, follow_redirects=False) + assert response.status_code == 303 + return client + + +def epub_id(sessions) -> int: + with sessions() as s: + return s.scalar(select(DocumentRow.id).where(DocumentRow.format == "epub")) + + +def test_healthz_is_public(client): + assert client.get("/healthz").json() == {"status": "ok"} + + +def test_pages_require_login(client): + response = client.get("/b/1?x=1", follow_redirects=False) + assert response.status_code == 303 + assert response.headers["location"] == "/login?next=/b/1%3Fx%3D1" + + +def test_wrong_password_then_lockout(client): + for _ in range(10): + assert "Wrong password" in client.post("/login", data={"password": "nope"}).text + blocked = client.post("/login", data={"password": "correct horse"}) + assert "Too many attempts" in blocked.text + + +def test_login_redirect_stays_on_site(client): + response = client.post( + "/login", data={"password": "correct horse", "next": "//evil.example"}, follow_redirects=False + ) + assert response.headers["location"] == "/" + + +def test_cross_site_post_is_blocked(logged_in): + response = logged_in.post("/rescan", headers={"Origin": "https://evil.example"}) + assert response.status_code == 403 + + +def test_library_lists_books(logged_in, library): + page = logged_in.get("/") + assert page.status_code == 200 + assert "The Test Book" in page.text and "Ada Tester" in page.text and "Playbook" in page.text + assert page.headers["Content-Security-Policy"].startswith("default-src 'self'") + assert "The Test Book" in logged_in.get("/?q=ada").text + assert "The Test Book" not in logged_in.get("/?q=playbook").text + + +def test_summarize_flow(logged_in, library, sessions): + doc_id = epub_id(sessions) + book = logged_in.get(f"/b/{doc_id}") + assert "Chapter 2" in book.text and "Summarize all remaining (3)" in book.text + + response = logged_in.post(f"/b/{doc_id}/summarize", data={"sections": "1"}, follow_redirects=False) + assert response.headers["location"] == f"/b/{doc_id}#s1" + assert "queued" in logged_in.get(f"/b/{doc_id}").text + assert 'http-equiv="refresh"' in logged_in.get(f"/b/{doc_id}/s/1").text + + while library.run_once(): + pass + section = logged_in.get(f"/b/{doc_id}/s/1") + assert "summary #1" in section.text + assert "Chapter 1" in section.text and "Chapter 3" in section.text # prev / next links + + logged_in.post(f"/b/{doc_id}/summarize", data={"sections": "all"}) + with sessions() as s: + queued = s.scalars(select(Job).where(Job.kind == "summarize", Job.status == "queued")).all() + assert sorted(j.payload["position"] for j in queued) == [0, 2] # chapter 2 is already done + + +def test_summary_html_is_escaped(logged_in, library, sessions, settings): + doc_id = epub_id(sessions) + with sessions() as s: + doc = s.get(DocumentRow, doc_id) + section = s.scalar( + select(SectionRow).where(SectionRow.document_id == doc_id, SectionRow.position == 0) + ) + key = cache_key("section", section.text_sha256, doc.title, section.title, settings.model) + PostgresCache(sessions).set(key, {"text": "**Bold** "}) + + page = logged_in.get(f"/b/{doc_id}/s/0").text + assert "Bold" in page + assert "