From dff3a6b71a4a3638d2b25a3d3a0d9277e784b9dd Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 00:08:12 +0800 Subject: [PATCH 1/6] =?UTF-8?q?feat(core):=20=E5=8E=9F=E5=AD=90=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=20Source=20=E6=89=B9=E6=AC=A1=E5=B9=B6=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=20B=E7=AB=99=E6=94=B6=E8=97=8F=E8=B5=84=E6=96=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增原生身份映射、配置与进度 CAS 及 Source 独占锚点迁移 - 提供 B站 Python 只读发行版与词法索引投影 - 适配 SVC 开发运行合同并固定可用的镜像构建工具依赖 --- .changes/source-batch.added.md | 1 + Dockerfile | 5 +- app/database_contract/protocol.py | 22 + docs/30-unit-tdd/README.md | 1 + docs/30-unit-tdd/bilibili-extension.md | 17 + .../business-pipeline-and-authority.md | 20 + docs/40-deployment/development-environment.md | 4 +- extensions/bilibili/.changes/reader.added.md | 1 + extensions/bilibili/CHANGELOG.md | 5 + extensions/bilibili/LICENSE | 21 + extensions/bilibili/README.md | 33 ++ extensions/bilibili/__init__.py | 20 + extensions/bilibili/pyproject.toml | 30 ++ extensions/bilibili/resolver.py | 76 +++ extensions/bilibili/schema.py | 113 +++++ inkcre_core/database_contract/constants.py | 3 +- inkcre_core/schemas/__init__.py | 2 + .../schemas/source/collection_record.py | 31 ++ migrations/revision-integrity.json | 3 + ...b2c3d4e5a6_add_source_batch_persistence.py | 187 +++++++ .../f1b2c3d4e5a7_commit_source_batch_root.py | 303 +++++++++++ .../f1b2c3d4e5a8_preserve_source_anchor.py | 334 ++++++++++++ scripts/dev_database_provider.py | 15 +- svc.json | 112 +++-- .../bilibili/test_lexical_integration.py | 51 ++ tests/extensions/bilibili/test_reader.py | 129 +++++ .../test_source_batch_persistence.py | 475 ++++++++++++++++++ tests/test_dev_database_provider.py | 55 ++ 28 files changed, 2022 insertions(+), 47 deletions(-) create mode 100644 .changes/source-batch.added.md create mode 100644 docs/30-unit-tdd/bilibili-extension.md create mode 100644 extensions/bilibili/.changes/reader.added.md create mode 100644 extensions/bilibili/CHANGELOG.md create mode 100644 extensions/bilibili/LICENSE create mode 100644 extensions/bilibili/README.md create mode 100644 extensions/bilibili/__init__.py create mode 100644 extensions/bilibili/pyproject.toml create mode 100644 extensions/bilibili/resolver.py create mode 100644 extensions/bilibili/schema.py create mode 100644 inkcre_core/schemas/source/collection_record.py create mode 100644 migrations/versions/f1b2c3d4e5a6_add_source_batch_persistence.py create mode 100644 migrations/versions/f1b2c3d4e5a7_commit_source_batch_root.py create mode 100644 migrations/versions/f1b2c3d4e5a8_preserve_source_anchor.py create mode 100644 tests/extensions/bilibili/test_lexical_integration.py create mode 100644 tests/extensions/bilibili/test_reader.py create mode 100644 tests/migrations/test_source_batch_persistence.py create mode 100644 tests/test_dev_database_provider.py diff --git a/.changes/source-batch.added.md b/.changes/source-batch.added.md new file mode 100644 index 00000000..869ca02e --- /dev/null +++ b/.changes/source-batch.added.md @@ -0,0 +1 @@ +数据库协议新增按原生身份幂等保存 Source 批次的能力；信息图、Source 进度与可选根入口原子提交，支持独立 Web Peer 的采集执行。 diff --git a/Dockerfile b/Dockerfile index 742f479d..5ed425d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,9 @@ ENV PDM_CHECK_UPDATE=false \ PDM_VENV_IN_PROJECT=1 WORKDIR /app -# PDM 2.28 requires the httpx client replaced in unearth 0.19. -RUN pip install --no-cache-dir "pdm==${PDM_VERSION}" "unearth==0.18.3" +# Keep the installer on the httpx API used by PDM 2.28. +RUN pip install --no-cache-dir "pdm==${PDM_VERSION}" "unearth==0.18.3" "hishel==1.3.0" \ + && python -c "from pdm.models.session import PDMPyPIClient" COPY pyproject.toml pdm.lock README.md LICENSE ./ COPY inkcre_core/ inkcre_core/ diff --git a/app/database_contract/protocol.py b/app/database_contract/protocol.py index 1b20d734..bf1b37fd 100644 --- a/app/database_contract/protocol.py +++ b/app/database_contract/protocol.py @@ -18,6 +18,26 @@ POSTGREST_OCTET_STREAM_TYPE = f'{PROTOCOL_SCHEMA}."application/octet-stream"' PROTOCOL_FUNCTIONS: dict[str, dict[str, object]] = { + "commit_source_batch": { + "arguments": [ + {"name": "source_id", "type": {"kind": "number", "format": "integer"}}, + {"name": "namespace", "type": {"kind": "string"}}, + *[ + {"name": name, "type": {"kind": "json"}} + for name in ( + "expected_config", + "expected_state", + "next_state", + "records", + "relations", + ) + ], + {"name": "root_key", "type": {"kind": "string"}, "has_default": True}, + ], + "returns": {"kind": "json"}, + "returns_set": False, + "volatility": "volatile", + }, "create_storage_blob": { "arguments": [ { @@ -76,6 +96,8 @@ def _database_type_name(type_document: dict[str, object]) -> str: if isinstance(format_, str): return format_ kind = type_document.get("kind") + if kind == "json": + return "jsonb" if kind == "string": return "text" if kind == "boolean": diff --git a/docs/30-unit-tdd/README.md b/docs/30-unit-tdd/README.md index f889824e..2cf3e0ca 100644 --- a/docs/30-unit-tdd/README.md +++ b/docs/30-unit-tdd/README.md @@ -14,6 +14,7 @@ This directory owns expensive internal design truth for logical units delivered | [memos-extension.md](memos-extension.md) | Memos adapter identity, graph grammar, persistence, and failure boundaries | | [capture-extension.md](capture-extension.md) | Capture submission identity, original materials, shared graph, and atomic recovery | | [local-files-extension.md](local-files-extension.md) | 本地文件资料、原件状态与正文的读取分发 | +| [bilibili-extension.md](bilibili-extension.md) | B 站共享收藏图的 Python 读取与词法投影边界 | | [rss-extension.md](rss-extension.md) | RSS adapter identity, collection lifecycle, reconciliation, and materialization | | [twitter-extension.md](twitter-extension.md) | Twitter 共同身份、Bookmark 过渡、事务合并和按需媒体读取 | | [security-model.md](security-model.md) | Core-py realization of the shared security boundary model | diff --git a/docs/30-unit-tdd/bilibili-extension.md b/docs/30-unit-tdd/bilibili-extension.md new file mode 100644 index 00000000..a37f5482 --- /dev/null +++ b/docs/30-unit-tdd/bilibili-extension.md @@ -0,0 +1,17 @@ +# B 站资料读取 + +当 Python Peer 需要读取或索引 Web Peer 已采集的 B 站资料时，启用 `inkcre/bilibili` 的 Python Distribution。它与 Web Distribution 解释同一份共享图，不拥有采集任务或浏览器会话。 + +## 读取与索引边界 + +`extensions/bilibili/schema.py` 验证四类 v1 原生记录，拒绝未知字段、显式 null、非字符串身份和错误类型。原生身份映射由 `collection_records` 拥有；记录之间的关联仅在 Relations，不在 Python 另存引用或内容副本。 + +Resolver 只从已持久化内容产生 Block-local 标签与文本。视频与收藏夹投影包括已保存的标题、简介，未知标题回退到 aid、BVID 或列表 ID。词法维护使用既有 `LexicalRetrievalManager` 和普通维护 Job；插件不直接写派生检索表，也不递归抓取相关视频以补全文本。 + +直接关系可通过 Core 的 Resolver 图接口读取。收藏条目的可用性与收藏时间独立于视频，避免不同收藏夹的观察互相覆盖。失效占位文字不得在采集阶段抹去旧标题；读取阶段如实解释已保存的较完整记录，不尝试恢复源站内容。 + +## Runtime 与交付 + +标准 Python Extension entry point 只装配 Resolver，没有 Source、HTTP 路由、Peer inbound 或私有后台任务。安装与启停沿用原有 Host；Python 类注册随进程保留，停用不模拟卸载已 import 的模块，本插件没有额外的 active effect。Host SDK 范围以 `app.version.CORE_VERSION` 的 `core-py` 边界为准，不把 `inkcre-core` 项目版本当成 Host 版本。 + +内容契约和用户使用范围见 [插件说明](../../extensions/bilibili/README.md)。发布、安装和精确版本选择继续遵守 [First-party Extension Distribution](../40-deployment/first-party-extension-distribution.md)。 diff --git a/docs/30-unit-tdd/business-pipeline-and-authority.md b/docs/30-unit-tdd/business-pipeline-and-authority.md index ea1b7778..c0dea070 100644 --- a/docs/30-unit-tdd/business-pipeline-and-authority.md +++ b/docs/30-unit-tdd/business-pipeline-and-authority.md @@ -266,6 +266,26 @@ HTTP 422。普通 REST 的请求输入、错误位置和内容表示见 [rest-in - Agent definition 是 shared database fact；Thread state、bound Tool handlers 与正在执行的 Turn 是 peer-local runtime fact。Agent definition 更新不会改写已经创建的 Thread snapshot。 +## Source 批次持久化 + +浏览器等非 Python 执行者通过数据库协议的 `commit_source_batch` 提交原生记录、稳定关系和 Source 进度。 +`collection_records` 只拥有 `(namespace, native_key) → Block` 的身份映射；语义内容仍由 Block 和 Relation +拥有，不进入 Source.state。namespace 是采集扩展选择的业务身份域，不是执行 Peer 或某次 Job 的身份。 + +一次调用先锁定 Source 并核对调用者观察到的完整 config/state，再取得 namespace 事务锁，串行化同一身份域 +的跨 Source 写入。记录复用既有 Block，顶层合并 JSON 对象，缺省字段保留；resolver 不一致、非 inline +内容或不合法端点拒绝整批。关系按精确端点和稳定 content 去重追加，不清理用户关系或未见条目。 +图与 next_state 在同一事务提交。可选 root_key 指向同 namespace 的业务根；Source.block 仍是该 Source +独占的 `core.source.v1` 锚点，不能直接指向跨 Source 共用的业务实体。事务内建立或刷新锚点的 +`{id,type,nickname}`，并追加锚点到业务根的 `collects` 关系；回包的 source_block 给出实际锚点 ID。 +省略 root_key 保留既有入口，不改写非 Source resolver 的 Block。CAS 冲突为 `PT409`，结果未知时调用者 +重新观察原 Source，不自动重放。 + +namespace 锁将冲突范围保持在扩展的身份域内，代价是同域批次串行；每次最多 500 条记录、2000 条关系和 +4 MiB JSON 输入，不能把无界归档作为单个事务。位置、收藏时间等可变成员事实由扩展的独立成员记录解释， +不借变化的 Relation.content 隐式创建新的关系身份。精确输入、DDL 和准入签名分别由 migration、metadata +与 `app/database_contract/protocol.py` 维护；本节不另建 DTO 权威。 + ## Non-Goals 本文件不承接以下内容： diff --git a/docs/40-deployment/development-environment.md b/docs/40-deployment/development-environment.md index 7523746f..7c9773f8 100644 --- a/docs/40-deployment/development-environment.md +++ b/docs/40-deployment/development-environment.md @@ -12,7 +12,9 @@ Python selector and PDM version. ## Local Startup -The supported complete runtime is an SVC worktree capability: +The supported complete runtime is an SVC worktree capability. `svc.json` declares schema 5 +resources, actions and readiness observations; use a SVC CLI that accepts this schema. The +database action keeps the worktree identity and delegates provisioning to `scripts/dev_database.py`: ```bash svc status . --json diff --git a/extensions/bilibili/.changes/reader.added.md b/extensions/bilibili/.changes/reader.added.md new file mode 100644 index 00000000..3e5ead6d --- /dev/null +++ b/extensions/bilibili/.changes/reader.added.md @@ -0,0 +1 @@ +新增 B 站收藏夹的 Python 读取 Distribution，解释同一插件 Web 采集的账号、收藏夹、视频和收藏条目 v1 图，为现有词法索引提供标题与简介投影；不采集源站、不保存视频二进制。 diff --git a/extensions/bilibili/CHANGELOG.md b/extensions/bilibili/CHANGELOG.md new file mode 100644 index 00000000..93ee0cf7 --- /dev/null +++ b/extensions/bilibili/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +此文件由本 Distribution 的 Towncrier 发布片段生成。 + + diff --git a/extensions/bilibili/LICENSE b/extensions/bilibili/LICENSE new file mode 100644 index 00000000..94e962af --- /dev/null +++ b/extensions/bilibili/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 InKCre contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/extensions/bilibili/README.md b/extensions/bilibili/README.md new file mode 100644 index 00000000..200b0757 --- /dev/null +++ b/extensions/bilibili/README.md @@ -0,0 +1,33 @@ +# B 站收藏夹读取 + +`inkcre/bilibili` 的 Python Distribution 解释共享信息库中的 B 站 v1 资料，为 Python Peer 提供读取与现有词法检索所需的文本投影。采集由同一插件的 Web Distribution 借用用户选择的浏览器完成；Python 读取不需要该浏览器在线，也不接触 B 站账号或 cookie。 + +## 使用 + +在需要解释或索引 B 站资料的 Python Peer 上安装并启用本 Distribution。Web 采集得到的账号、收藏夹、视频和收藏条目随后可由普通 Resolver 读取。现有 `core.feature_retrieval.lexical.maintain.v1` Job 负责维护索引；本插件只提供标签和文本解释，不另外写一套索引。 + +视频文本包含已保存的标题与简介，收藏夹文本包含名称与简介。未知标题使用原生身份作为可读回退；缺省字段表示未知。记录的直接关系继续通过普通图接口读取，包含收藏夹归属、成员与 UP 主。 + +## 持久契约 + +本版解释四个精确 Resolver ID： + +- `extensions.bilibili.user.v1`：B 站账号，含 MID 和实际取得的名称。 +- `extensions.bilibili.folder.v1`：收藏夹，含完整列表 ID、名称、可见性及来源报告数量。 +- `extensions.bilibili.video.v1`：视频，以 aid 为身份，BVID 为已知别名。 +- `extensions.bilibili.membership.v1`：收藏条目的可用性、收藏时间和本次页面观察位置。 + +图关系为 `账号 —owns→ 收藏夹 —contains→ 收藏条目 —item→ 视频 —author→ UP 主`。同一视频可由多个收藏条目引用。成员记录中的页码和页内位置是采集观察事实，不承诺原站的实时绝对顺序。 + +本 Distribution 不注册 Source，不下载视频二进制、字幕、封面或其他远程资料，不恢复失效视频，也不修改源站收藏。封面 URL 仅作为已保存事实被解释。原生身份去重与批次状态原子提交由采集端和数据库协议完成。 + +## 兼容与验证 + +Host SDK 为 `core-py >=0.3.4 <0.4.0`；该版本来自 Python App 的 Host SDK 版本，区别于独立 `inkcre-core` SDK 项目的版本号。 + +```sh +pdm run python -m pytest -q tests/extensions/bilibili +pdm run python -m build --wheel extensions/bilibili +``` + +单元测试使用与 Web v1 形状一致的虚构资料，并调用现有词法投影入口。真实浏览器采集、数据库持久化和跨 Peer 检索由主任务验收记录报告。 diff --git a/extensions/bilibili/__init__.py b/extensions/bilibili/__init__.py new file mode 100644 index 00000000..07b7b7c3 --- /dev/null +++ b/extensions/bilibili/__init__.py @@ -0,0 +1,20 @@ +"""B 站插件的 Python 读取 Distribution。""" + +import pydantic + +from app.business.extension import ExtensionBase + + +class BilibiliConfig(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + + +class Extension( + ExtensionBase[BilibiliConfig], ext_id="bilibili", config_cls=BilibiliConfig +): + @classmethod + def _init_resolvers(cls) -> None: + from . import resolver as resolver_module # noqa: F401 + + +__all__ = ["BilibiliConfig", "Extension"] diff --git a/extensions/bilibili/pyproject.toml b/extensions/bilibili/pyproject.toml new file mode 100644 index 00000000..581ff554 --- /dev/null +++ b/extensions/bilibili/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=80,<81"] +build-backend = "setuptools.build_meta" + +[project] +name = "inkcre-ext-bilibili" +version = "0.1.0" +description = "Read shared Bilibili favorites in Python peers" +requires-python = ">=3.12,<3.13" +readme = "README.md" +license = "MIT" +license-files = ["LICENSE"] +dependencies = [ + "pydantic>=2.11.7,<3.0.0", +] + +[project.entry-points."inkcre.core.extensions"] +bilibili = "extensions.bilibili:Extension" + +[tool.inkcre-extension] +name = "inkcre/bilibili" +nickname = "B 站收藏夹" +host-sdk = "core-py" +host-sdk-version = ">=0.3.4 <0.4.0" + +[tool.setuptools] +packages = ["extensions.bilibili"] + +[tool.setuptools.package-dir] +"extensions.bilibili" = "." diff --git a/extensions/bilibili/resolver.py b/extensions/bilibili/resolver.py new file mode 100644 index 00000000..359fa4cf --- /dev/null +++ b/extensions/bilibili/resolver.py @@ -0,0 +1,76 @@ +"""读取共享收藏资料；词法索引归现有 LexicalRetrievalManager 维护。""" + +from __future__ import annotations + +import typing + +from inkcre_core.business.info_base.resolver import Resolver, TextProjectionContext + +from .schema import ( + FOLDER_RESOLVER, + MEMBERSHIP_RESOLVER, + USER_RESOLVER, + VIDEO_RESOLVER, + BilibiliRecord, + Folder, + Membership, + User, + Video, + record_title, +) + + +RecordT = typing.TypeVar("RecordT", bound=BilibiliRecord) + + +class _Reader(typing.Generic[RecordT]): + content_model: type[RecordT] + + async def _get_solved_content( + self, *, refresh: bool = False, materialize_missing: bool = True + ) -> RecordT: + del materialize_missing + resolver = typing.cast(Resolver[RecordT, str], self) + return self.content_model.model_validate_json( + await resolver.get_raw_content(refresh=refresh) + ) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, materialize_missing + resolver = typing.cast(Resolver[RecordT, str], self) + value = await resolver.get_solved_content(refresh=refresh, materialize_missing=False) + description = value.description if isinstance(value, (Folder, Video)) else None + return "\n".join(part for part in (record_title(value), description) if part) + + async def get_label(self, *, refresh: bool = False) -> str: + resolver = typing.cast(Resolver[RecordT, str], self) + return record_title( + await resolver.get_solved_content(refresh=refresh, materialize_missing=False) + ) + + +class UserResolver(_Reader[User], Resolver[User, str], rso_type=USER_RESOLVER): + content_model = User + + +class FolderResolver(_Reader[Folder], Resolver[Folder, str], rso_type=FOLDER_RESOLVER): + content_model = Folder + + +class VideoResolver(_Reader[Video], Resolver[Video, str], rso_type=VIDEO_RESOLVER): + content_model = Video + + +class MembershipResolver( + _Reader[Membership], Resolver[Membership, str], rso_type=MEMBERSHIP_RESOLVER +): + content_model = Membership + + +__all__ = ["FolderResolver", "MembershipResolver", "UserResolver", "VideoResolver"] diff --git a/extensions/bilibili/schema.py b/extensions/bilibili/schema.py new file mode 100644 index 00000000..7f95f937 --- /dev/null +++ b/extensions/bilibili/schema.py @@ -0,0 +1,113 @@ +"""B 站 v1 持久图记录，与 MF Distribution 共用语义。""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import ( + AnyUrl, + AwareDatetime, + BaseModel, + ConfigDict, + Field, + StrictBool, + StrictInt, + field_validator, + model_validator, +) + + +USER_RESOLVER = "extensions.bilibili.user.v1" +FOLDER_RESOLVER = "extensions.bilibili.folder.v1" +VIDEO_RESOLVER = "extensions.bilibili.video.v1" +MEMBERSHIP_RESOLVER = "extensions.bilibili.membership.v1" + +NativeID = Annotated[str, Field(strict=True, pattern=r"^[1-9][0-9]*$")] +NonnegativeInt = Annotated[StrictInt, Field(ge=0)] +PositiveInt = Annotated[StrictInt, Field(ge=1)] +UTCInstant = Annotated[AwareDatetime, Field(strict=False)] + + +class Record(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + @model_validator(mode="before") + @classmethod + def sparse_not_null(cls, value: object) -> object: + # The MF contract uses omitted fields for unknown facts, never authored nulls. + if isinstance(value, dict) and any(item is None for item in value.values()): + raise ValueError("未知字段应省略，不能保存 null") + return value + + +class User(Record): + mid: NativeID + name: str | None = None + avatar_url: AnyUrl | None = None + + +class Folder(Record): + id: NativeID + title: str | None = None + description: str | None = None + is_private: StrictBool | None = None + reported_count: NonnegativeInt | None = None + + +class Video(Record): + aid: NativeID + bvid: Annotated[str, Field(pattern=r"^BV[0-9A-Za-z]{10}$")] | None = None + title: str | None = None + description: str | None = None + duration_seconds: NonnegativeInt | None = None + published_at: UTCInstant | None = None + cover_url: AnyUrl | None = None + + @field_validator("published_at", mode="before") + @classmethod + def utc_string(cls, value: object) -> object: + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError("来源时间必须是 UTC ISO 字符串") + return value + + +class Membership(Record): + availability: Literal["available", "unavailable", "unknown"] + page_number: PositiveInt | None = None + position_in_page: NonnegativeInt | None = None + favored_at: UTCInstant | None = None + observed_at: UTCInstant + + @field_validator("favored_at", "observed_at", mode="before") + @classmethod + def utc_string(cls, value: object) -> object: + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError("来源时间必须是 UTC ISO 字符串") + return value + + +BilibiliRecord = User | Folder | Video | Membership + + +def record_title(value: BilibiliRecord) -> str: + if isinstance(value, User): + return value.name or f"UID {value.mid}" + if isinstance(value, Folder): + return value.title or f"收藏夹 {value.id}" + if isinstance(value, Video): + return value.title or value.bvid or f"av{value.aid}" + return "已失效的收藏条目" if value.availability == "unavailable" else "收藏条目" + + +__all__ = [ + "FOLDER_RESOLVER", + "MEMBERSHIP_RESOLVER", + "USER_RESOLVER", + "VIDEO_RESOLVER", + "BilibiliRecord", + "Folder", + "Membership", + "User", + "Video", + "record_title", +] diff --git a/inkcre_core/database_contract/constants.py b/inkcre_core/database_contract/constants.py index beb328a3..80979f4f 100644 --- a/inkcre_core/database_contract/constants.py +++ b/inkcre_core/database_contract/constants.py @@ -4,7 +4,7 @@ CONTRACT_FORMAT = 1 -CONTRACT_REVISION = "extension-distribution-open-v1" +CONTRACT_REVISION = "source-batch-persistence-v1" PROTOCOL_SCHEMA = "inkcre" INTERNAL_SCHEMA = "inkcre_internal" @@ -36,6 +36,7 @@ "block_embeddings", "block_lexical_records", "blocks", + "collection_records", "peers", "configs", "extensions", diff --git a/inkcre_core/schemas/__init__.py b/inkcre_core/schemas/__init__.py index c7eb0f62..f9e5038d 100644 --- a/inkcre_core/schemas/__init__.py +++ b/inkcre_core/schemas/__init__.py @@ -23,6 +23,7 @@ "DeploymentConfigModel", "SinkModel", "SinkTypeModel", + "CollectionRecordModel", ] import sqlalchemy.orm @@ -54,3 +55,4 @@ from .deployment_config import DeploymentConfigModel from .lexical_retrieval import BlockLexicalRecordModel from .sink import SinkModel, SinkTypeModel +from .source.collection_record import CollectionRecordModel diff --git a/inkcre_core/schemas/source/collection_record.py b/inkcre_core/schemas/source/collection_record.py new file mode 100644 index 00000000..b7e116a0 --- /dev/null +++ b/inkcre_core/schemas/source/collection_record.py @@ -0,0 +1,31 @@ +"""Native identity lookup; Block content remains the information authority.""" + +import sqlalchemy +import sqlmodel + + +class CollectionRecordModel(sqlmodel.SQLModel, table=True): + __tablename__ = "collection_records" # type: ignore + __table_args__ = ( + sqlalchemy.CheckConstraint( + "length(namespace) BETWEEN 1 AND 200", name="collection_record_namespace" + ), + sqlalchemy.CheckConstraint( + "length(native_key) BETWEEN 1 AND 1000", name="collection_record_key" + ), + ) + + namespace: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) + ) + native_key: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) + ) + block: int = sqlmodel.Field( + sa_column=sqlalchemy.Column( + sqlalchemy.Integer, + sqlalchemy.ForeignKey("blocks.id", ondelete="CASCADE", onupdate="CASCADE"), + nullable=False, + index=True, + ) + ) diff --git a/migrations/revision-integrity.json b/migrations/revision-integrity.json index 7b0dece0..47d2b216 100644 --- a/migrations/revision-integrity.json +++ b/migrations/revision-integrity.json @@ -28,6 +28,9 @@ "e1f4a5b6c7d8_migrate_memos_attachment_v2.py": "5f3691149e06b28378f79eed7eaacd1c415ee6c92fb83b3b5745a429346d5e01", "e5a01f9e69ef_init.py": "a10de6c0818abc593951a34e51a974ffcd999f60ba5afc441f87ac6cef8d869e", "e6a1b2c3d4f5_extension_peer_participation_v2.py": "bf71d955b1e3e27015f412815d020666157eda6af78ed0472ca2b7d38ada872f", + "f1b2c3d4e5a6_add_source_batch_persistence.py": "f80ebf02ea7475ad2935aabd39904f5062696572847deebd1d1f2f7fd5eea7cd", + "f1b2c3d4e5a7_commit_source_batch_root.py": "0ba844592cecdc1b3847ab1f32f59e3d7f7cafe335f97923ee36a0e6d8f3037e", + "f1b2c3d4e5a8_preserve_source_anchor.py": "0d86aa4f01040a834a8e1dc21fb85c0f60d555115fcaf8b24ba7cbf20d64b6ce", "f2a6c8e4b1d7_add_extension_registry_deployment_state.py": "a2cf3059f87e17292baf9fd10a1acb8b26ccb928845dbe4bec95b374164cd46d", "f2c8a6d1e4b7_add_postgresql_binary_storage.py": "1137323490b324da3f1672cf5a3a8b5edb1e6a258cc10129926bf6e8d232a111", "f7a8b9c0d1e2_add_deployment_configs.py": "9bb58b1d22dccef5817e28df8bcb88ab6edfff70a6823d80b3637752899188fe" diff --git a/migrations/versions/f1b2c3d4e5a6_add_source_batch_persistence.py b/migrations/versions/f1b2c3d4e5a6_add_source_batch_persistence.py new file mode 100644 index 00000000..01060214 --- /dev/null +++ b/migrations/versions/f1b2c3d4e5a6_add_source_batch_persistence.py @@ -0,0 +1,187 @@ +"""Add native collection identities and atomic Source graph batches. + +Revision ID: f1b2c3d4e5a6 +Revises: dcf44bc9af73 +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "f1b2c3d4e5a6" +down_revision = "dcf44bc9af73" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "collection_records", + sa.Column("namespace", sa.Text(), nullable=False), + sa.Column("native_key", sa.Text(), nullable=False), + sa.Column("block", sa.Integer(), nullable=False), + sa.CheckConstraint( + "length(namespace) BETWEEN 1 AND 200", name="collection_record_namespace" + ), + sa.CheckConstraint( + "length(native_key) BETWEEN 1 AND 1000", name="collection_record_key" + ), + sa.ForeignKeyConstraint( + ["block"], ["inkcre.blocks.id"], ondelete="CASCADE", onupdate="CASCADE" + ), + sa.PrimaryKeyConstraint("namespace", "native_key"), + schema="inkcre", + ) + op.create_index( + "ix_inkcre_collection_records_block", "collection_records", ["block"], schema="inkcre" + ) + op.execute(""" + CREATE FUNCTION inkcre.commit_source_batch( + source_id integer, + namespace text, + expected_config jsonb, + expected_state jsonb, + next_state jsonb, + records jsonb, + relations jsonb + ) RETURNS jsonb + LANGUAGE plpgsql VOLATILE SECURITY INVOKER + SET search_path = pg_catalog + AS $function$ + DECLARE + source_config jsonb; + source_state jsonb; + item jsonb; + block_row record; + block_id integer; + from_id integer; + to_id integer; + result_blocks jsonb := '{}'::jsonb; + BEGIN + IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 + OR namespace !~ '[^[:space:]]' + OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' + OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(records) IS DISTINCT FROM 'array' + OR jsonb_typeof(relations) IS DISTINCT FROM 'array' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; + END IF; + IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 + OR octet_length((expected_config::text || expected_state::text || next_state::text + || records::text || relations::text)) > 4194304 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; + END IF; + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements(records) AS entry + GROUP BY entry ->> 'key' HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; + END IF; + + SELECT source.config, source.state INTO source_config, source_state + FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; + END IF; + IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config + OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state + THEN + RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; + END IF; + + -- Serialize identities across Sources, not just executions of this Source. + PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); + FOR item IN SELECT value FROM jsonb_array_elements(records) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' + OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 + OR (item ->> 'key') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' + OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 + OR (item ->> 'resolver') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; + END IF; + SELECT block.* INTO block_row + FROM inkcre.collection_records AS identity + JOIN inkcre.blocks AS block ON block.id = identity.block + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' + FOR UPDATE OF block; + IF FOUND THEN + IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' + OR block_row.storage IS NOT NULL + OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; + END IF; + block_id := block_row.id; + UPDATE inkcre.blocks AS block + SET content = (block_row.content::jsonb || (item -> 'content'))::text + WHERE block.id = block_id; + ELSE + INSERT INTO inkcre.blocks (storage, resolver, content) + VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) + RETURNING id INTO block_id; + INSERT INTO inkcre.collection_records (namespace, native_key, block) + VALUES ($2, item ->> 'key', block_id); + END IF; + result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); + END LOOP; + + FOR item IN SELECT value FROM jsonb_array_elements(relations) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' + OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; + END IF; + SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; + SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; + IF from_id IS NULL OR to_id IS NULL THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; + END IF; + INSERT INTO inkcre.relations (from_, to_, content) + SELECT from_id, to_id, item ->> 'content' + WHERE NOT EXISTS ( + SELECT 1 FROM inkcre.relations AS relation + WHERE relation.from_ = from_id AND relation.to_ = to_id + AND relation.content = item ->> 'content' + ); + END LOOP; + UPDATE inkcre.sources AS source SET state = next_state WHERE source.id = source_id; + RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state); + END; + $function$; + """) + op.execute( + "REVOKE ALL ON FUNCTION inkcre.commit_source_batch(integer, text, jsonb, jsonb, jsonb, jsonb, jsonb) FROM PUBLIC" + ) + op.execute( + "UPDATE inkcre_internal.contract_state " + "SET contract_revision='source-batch-persistence-v1',updated_at=CURRENT_TIMESTAMP " + "WHERE singleton" + ) + + +def downgrade() -> None: + op.execute( + "DROP FUNCTION inkcre.commit_source_batch(integer, text, jsonb, jsonb, jsonb, jsonb, jsonb)" + ) + op.drop_table("collection_records", schema="inkcre") + op.execute( + "UPDATE inkcre_internal.contract_state " + "SET contract_revision='extension-distribution-open-v1',updated_at=CURRENT_TIMESTAMP " + "WHERE singleton" + ) diff --git a/migrations/versions/f1b2c3d4e5a7_commit_source_batch_root.py b/migrations/versions/f1b2c3d4e5a7_commit_source_batch_root.py new file mode 100644 index 00000000..652cc73c --- /dev/null +++ b/migrations/versions/f1b2c3d4e5a7_commit_source_batch_root.py @@ -0,0 +1,303 @@ +"""Commit the optional Source graph root with its batch. + +Revision ID: f1b2c3d4e5a7 +Revises: f1b2c3d4e5a6 +""" + +from alembic import op + + +revision = "f1b2c3d4e5a7" +down_revision = "f1b2c3d4e5a6" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + "DROP FUNCTION inkcre.commit_source_batch(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb)" + ) + op.execute(""" + CREATE FUNCTION inkcre.commit_source_batch( + source_id integer, + namespace text, + expected_config jsonb, + expected_state jsonb, + next_state jsonb, + records jsonb, + relations jsonb, + root_key text DEFAULT NULL + ) RETURNS jsonb + LANGUAGE plpgsql VOLATILE SECURITY INVOKER + SET search_path = pg_catalog + AS $function$ + DECLARE + source_config jsonb; + source_state jsonb; + item jsonb; + block_row record; + block_id integer; + from_id integer; + to_id integer; + root_block_id integer; + result_blocks jsonb := '{}'::jsonb; + BEGIN + IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 + OR namespace !~ '[^[:space:]]' + OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' + OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(records) IS DISTINCT FROM 'array' + OR jsonb_typeof(relations) IS DISTINCT FROM 'array' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; + END IF; + IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 + OR octet_length((expected_config::text || expected_state::text || next_state::text + || records::text || relations::text)) > 4194304 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; + END IF; + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements(records) AS entry + GROUP BY entry ->> 'key' HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; + END IF; + + SELECT source.config, source.state INTO source_config, source_state + FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; + END IF; + IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config + OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state + THEN + RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; + END IF; + + -- Serialize identities across Sources, not just executions of this Source. + PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); + FOR item IN SELECT value FROM jsonb_array_elements(records) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' + OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 + OR (item ->> 'key') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' + OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 + OR (item ->> 'resolver') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; + END IF; + SELECT block.* INTO block_row + FROM inkcre.collection_records AS identity + JOIN inkcre.blocks AS block ON block.id = identity.block + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' + FOR UPDATE OF block; + IF FOUND THEN + IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' + OR block_row.storage IS NOT NULL + OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; + END IF; + block_id := block_row.id; + UPDATE inkcre.blocks AS block + SET content = (block_row.content::jsonb || (item -> 'content'))::text + WHERE block.id = block_id; + ELSE + INSERT INTO inkcre.blocks (storage, resolver, content) + VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) + RETURNING id INTO block_id; + INSERT INTO inkcre.collection_records (namespace, native_key, block) + VALUES ($2, item ->> 'key', block_id); + END IF; + result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); + END LOOP; + + FOR item IN SELECT value FROM jsonb_array_elements(relations) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' + OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; + END IF; + SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; + SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; + IF from_id IS NULL OR to_id IS NULL THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; + END IF; + INSERT INTO inkcre.relations (from_, to_, content) + SELECT from_id, to_id, item ->> 'content' + WHERE NOT EXISTS ( + SELECT 1 FROM inkcre.relations AS relation + WHERE relation.from_ = from_id AND relation.to_ = to_id + AND relation.content = item ->> 'content' + ); + END LOOP; + IF root_key IS NOT NULL THEN + SELECT identity.block INTO root_block_id + FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = root_key; + IF root_block_id IS NULL THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source root identity is unknown'; + END IF; + END IF; + UPDATE inkcre.sources AS source + SET state = next_state, block = coalesce(root_block_id, source.block) + WHERE source.id = source_id; + RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state); + END; + $function$; + """) + op.execute( + "REVOKE ALL ON FUNCTION inkcre.commit_source_batch(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb,text) FROM PUBLIC" + ) + + +def downgrade() -> None: + op.execute( + "DROP FUNCTION inkcre.commit_source_batch(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb,text)" + ) + op.execute(""" + CREATE FUNCTION inkcre.commit_source_batch( + source_id integer, + namespace text, + expected_config jsonb, + expected_state jsonb, + next_state jsonb, + records jsonb, + relations jsonb + ) RETURNS jsonb + LANGUAGE plpgsql VOLATILE SECURITY INVOKER + SET search_path = pg_catalog + AS $function$ + DECLARE + source_config jsonb; + source_state jsonb; + item jsonb; + block_row record; + block_id integer; + from_id integer; + to_id integer; + result_blocks jsonb := '{}'::jsonb; + BEGIN + IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 + OR namespace !~ '[^[:space:]]' + OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' + OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(records) IS DISTINCT FROM 'array' + OR jsonb_typeof(relations) IS DISTINCT FROM 'array' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; + END IF; + IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 + OR octet_length((expected_config::text || expected_state::text || next_state::text + || records::text || relations::text)) > 4194304 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; + END IF; + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements(records) AS entry + GROUP BY entry ->> 'key' HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; + END IF; + + SELECT source.config, source.state INTO source_config, source_state + FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; + END IF; + IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config + OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state + THEN + RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; + END IF; + + -- Serialize identities across Sources, not just executions of this Source. + PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); + FOR item IN SELECT value FROM jsonb_array_elements(records) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' + OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 + OR (item ->> 'key') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' + OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 + OR (item ->> 'resolver') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; + END IF; + SELECT block.* INTO block_row + FROM inkcre.collection_records AS identity + JOIN inkcre.blocks AS block ON block.id = identity.block + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' + FOR UPDATE OF block; + IF FOUND THEN + IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' + OR block_row.storage IS NOT NULL + OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; + END IF; + block_id := block_row.id; + UPDATE inkcre.blocks AS block + SET content = (block_row.content::jsonb || (item -> 'content'))::text + WHERE block.id = block_id; + ELSE + INSERT INTO inkcre.blocks (storage, resolver, content) + VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) + RETURNING id INTO block_id; + INSERT INTO inkcre.collection_records (namespace, native_key, block) + VALUES ($2, item ->> 'key', block_id); + END IF; + result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); + END LOOP; + + FOR item IN SELECT value FROM jsonb_array_elements(relations) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' + OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; + END IF; + SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; + SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; + IF from_id IS NULL OR to_id IS NULL THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; + END IF; + INSERT INTO inkcre.relations (from_, to_, content) + SELECT from_id, to_id, item ->> 'content' + WHERE NOT EXISTS ( + SELECT 1 FROM inkcre.relations AS relation + WHERE relation.from_ = from_id AND relation.to_ = to_id + AND relation.content = item ->> 'content' + ); + END LOOP; + UPDATE inkcre.sources AS source SET state = next_state WHERE source.id = source_id; + RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state); + END; + $function$; + """) + op.execute( + "REVOKE ALL ON FUNCTION inkcre.commit_source_batch(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb) FROM PUBLIC" + ) diff --git a/migrations/versions/f1b2c3d4e5a8_preserve_source_anchor.py b/migrations/versions/f1b2c3d4e5a8_preserve_source_anchor.py new file mode 100644 index 00000000..56c56dff --- /dev/null +++ b/migrations/versions/f1b2c3d4e5a8_preserve_source_anchor.py @@ -0,0 +1,334 @@ +"""Preserve Source anchor identity while linking collected graph roots. + +Revision ID: f1b2c3d4e5a8 +Revises: f1b2c3d4e5a7 +""" + +from alembic import op + + +revision = "f1b2c3d4e5a8" +down_revision = "f1b2c3d4e5a7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + CREATE OR REPLACE FUNCTION inkcre.commit_source_batch( + source_id integer, + namespace text, + expected_config jsonb, + expected_state jsonb, + next_state jsonb, + records jsonb, + relations jsonb, + root_key text DEFAULT NULL + ) RETURNS jsonb + LANGUAGE plpgsql VOLATILE SECURITY INVOKER + SET search_path = pg_catalog + AS $function$ + DECLARE + source_config jsonb; + source_state jsonb; + source_type text; + source_nickname text; + source_anchor_id integer; + anchor_content jsonb; + anchor_row record; + item jsonb; + block_row record; + block_id integer; + from_id integer; + to_id integer; + root_block_id integer; + result_blocks jsonb := '{}'::jsonb; + BEGIN + IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 + OR namespace !~ '[^[:space:]]' + OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' + OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(records) IS DISTINCT FROM 'array' + OR jsonb_typeof(relations) IS DISTINCT FROM 'array' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; + END IF; + IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 + OR octet_length((expected_config::text || expected_state::text || next_state::text + || records::text || relations::text)) > 4194304 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; + END IF; + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements(records) AS entry + GROUP BY entry ->> 'key' HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; + END IF; + + SELECT source.config, source.state, source.type, source.nickname, source.block + INTO source_config, source_state, source_type, source_nickname, source_anchor_id + FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; + END IF; + IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config + OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state + THEN + RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; + END IF; + + -- Serialize identities across Sources, not just executions of this Source. + PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); + FOR item IN SELECT value FROM jsonb_array_elements(records) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' + OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 + OR (item ->> 'key') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' + OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 + OR (item ->> 'resolver') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; + END IF; + SELECT block.* INTO block_row + FROM inkcre.collection_records AS identity + JOIN inkcre.blocks AS block ON block.id = identity.block + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' + FOR UPDATE OF block; + IF FOUND THEN + IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' + OR block_row.storage IS NOT NULL + OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; + END IF; + block_id := block_row.id; + UPDATE inkcre.blocks AS block + SET content = (block_row.content::jsonb || (item -> 'content'))::text + WHERE block.id = block_id; + ELSE + INSERT INTO inkcre.blocks (storage, resolver, content) + VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) + RETURNING id INTO block_id; + INSERT INTO inkcre.collection_records (namespace, native_key, block) + VALUES ($2, item ->> 'key', block_id); + END IF; + result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); + END LOOP; + + FOR item IN SELECT value FROM jsonb_array_elements(relations) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' + OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; + END IF; + SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; + SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; + IF from_id IS NULL OR to_id IS NULL THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; + END IF; + INSERT INTO inkcre.relations (from_, to_, content) + SELECT from_id, to_id, item ->> 'content' + WHERE NOT EXISTS ( + SELECT 1 FROM inkcre.relations AS relation + WHERE relation.from_ = from_id AND relation.to_ = to_id + AND relation.content = item ->> 'content' + ); + END LOOP; + IF root_key IS NOT NULL THEN + SELECT identity.block INTO root_block_id + FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = root_key; + IF root_block_id IS NULL THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source root identity is unknown'; + END IF; + anchor_content := jsonb_build_object('id', source_id, 'type', source_type, 'nickname', source_nickname); + IF source_anchor_id IS NULL THEN + INSERT INTO inkcre.blocks (storage, resolver, content) + VALUES (NULL, 'core.source.v1', anchor_content::text) + RETURNING id INTO source_anchor_id; + ELSE + SELECT block.* INTO anchor_row + FROM inkcre.blocks AS block WHERE block.id = source_anchor_id FOR UPDATE; + IF NOT FOUND OR anchor_row.resolver IS DISTINCT FROM 'core.source.v1' + OR anchor_row.storage IS NOT NULL + OR jsonb_typeof(anchor_row.content::jsonb) IS DISTINCT FROM 'object' + OR (anchor_row.content::jsonb -> 'id') IS DISTINCT FROM to_jsonb(source_id) + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source anchor is incompatible'; + END IF; + UPDATE inkcre.blocks AS block SET content = anchor_content::text + WHERE block.id = source_anchor_id; + END IF; + INSERT INTO inkcre.relations (from_, to_, content) + SELECT source_anchor_id, root_block_id, 'collects' + WHERE NOT EXISTS ( + SELECT 1 FROM inkcre.relations AS relation + WHERE relation.from_ = source_anchor_id AND relation.to_ = root_block_id + AND relation.content = 'collects' + ); + END IF; + UPDATE inkcre.sources AS source + SET state = next_state, block = source_anchor_id + WHERE source.id = source_id; + RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state, 'source_block', source_anchor_id); + END; + $function$; + """) + + +def downgrade() -> None: + op.execute(""" + CREATE OR REPLACE FUNCTION inkcre.commit_source_batch( + source_id integer, + namespace text, + expected_config jsonb, + expected_state jsonb, + next_state jsonb, + records jsonb, + relations jsonb, + root_key text DEFAULT NULL + ) RETURNS jsonb + LANGUAGE plpgsql VOLATILE SECURITY INVOKER + SET search_path = pg_catalog + AS $function$ + DECLARE + source_config jsonb; + source_state jsonb; + item jsonb; + block_row record; + block_id integer; + from_id integer; + to_id integer; + root_block_id integer; + result_blocks jsonb := '{}'::jsonb; + BEGIN + IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 + OR namespace !~ '[^[:space:]]' + OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' + OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' + OR jsonb_typeof(records) IS DISTINCT FROM 'array' + OR jsonb_typeof(relations) IS DISTINCT FROM 'array' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; + END IF; + IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 + OR octet_length((expected_config::text || expected_state::text || next_state::text + || records::text || relations::text)) > 4194304 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; + END IF; + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements(records) AS entry + GROUP BY entry ->> 'key' HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; + END IF; + + SELECT source.config, source.state INTO source_config, source_state + FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; + END IF; + IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config + OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state + THEN + RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; + END IF; + + -- Serialize identities across Sources, not just executions of this Source. + PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); + FOR item IN SELECT value FROM jsonb_array_elements(records) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' + OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 + OR (item ->> 'key') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' + OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 + OR (item ->> 'resolver') !~ '[^[:space:]]' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; + END IF; + SELECT block.* INTO block_row + FROM inkcre.collection_records AS identity + JOIN inkcre.blocks AS block ON block.id = identity.block + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' + FOR UPDATE OF block; + IF FOUND THEN + IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' + OR block_row.storage IS NOT NULL + OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; + END IF; + block_id := block_row.id; + UPDATE inkcre.blocks AS block + SET content = (block_row.content::jsonb || (item -> 'content'))::text + WHERE block.id = block_id; + ELSE + INSERT INTO inkcre.blocks (storage, resolver, content) + VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) + RETURNING id INTO block_id; + INSERT INTO inkcre.collection_records (namespace, native_key, block) + VALUES ($2, item ->> 'key', block_id); + END IF; + result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); + END LOOP; + + FOR item IN SELECT value FROM jsonb_array_elements(relations) + LOOP + IF jsonb_typeof(item) IS DISTINCT FROM 'object' + OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb + OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' + OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' + OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 + THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; + END IF; + SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; + SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; + IF from_id IS NULL OR to_id IS NULL THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; + END IF; + INSERT INTO inkcre.relations (from_, to_, content) + SELECT from_id, to_id, item ->> 'content' + WHERE NOT EXISTS ( + SELECT 1 FROM inkcre.relations AS relation + WHERE relation.from_ = from_id AND relation.to_ = to_id + AND relation.content = item ->> 'content' + ); + END LOOP; + IF root_key IS NOT NULL THEN + SELECT identity.block INTO root_block_id + FROM inkcre.collection_records AS identity + WHERE identity.namespace = $2 AND identity.native_key = root_key; + IF root_block_id IS NULL THEN + RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source root identity is unknown'; + END IF; + END IF; + UPDATE inkcre.sources AS source + SET state = next_state, block = coalesce(root_block_id, source.block) + WHERE source.id = source_id; + RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state); + END; + $function$; + """) diff --git a/scripts/dev_database_provider.py b/scripts/dev_database_provider.py index bd6e7b42..e52914ef 100644 --- a/scripts/dev_database_provider.py +++ b/scripts/dev_database_provider.py @@ -23,8 +23,10 @@ BUILD_CONTEXT_PATHS = ( Path("Dockerfile"), Path("README.md"), + Path("LICENSE"), Path("alembic.ini"), Path("app"), + Path("inkcre_core"), Path("extensions"), Path("libs"), Path("migrations"), @@ -91,6 +93,17 @@ def _provision_environment( legacy_profile: str | None, ) -> dict[str, str]: dev = config.get("dev", {}) + resources = dev.get("resources") + if isinstance(resources, dict): + environment = ( + resources.get("database", {}).get("actions", {}).get("ensure", {}).get("env", {}) + ) + if not isinstance(environment, dict) or not all( + isinstance(name, str) and isinstance(value, str) + for name, value in environment.items() + ): + raise ValueError("database provision environment must contain string values") + return environment targets = dev.get("targets") if targets is None: profile = dev.get("profile") or legacy_profile @@ -201,7 +214,7 @@ def _compose_failure( compose_environment: Path, provider: DatabaseProvider, ) -> RuntimeError: - raw_diagnostics = error.stderr or error.stdout or b"" + raw_diagnostics = b"\n".join(value for value in (error.stderr, error.stdout) if value) diagnostics = raw_diagnostics.decode(errors="replace") try: for line in compose_environment.read_text().splitlines(): diff --git a/svc.json b/svc.json index f652b409..546f68d8 100644 --- a/svc.json +++ b/svc.json @@ -1,50 +1,78 @@ { - "schema_version": 4, + "schema_version": 5, "dev": { - "targets": { + "resources": { "database": { - "poll_interval": 1, - "probe": { - "argv": [ - "pdm", - "run", - "dev:database", - "probe", - "${dev.instance}" - ], - "cwd": ".", - "kind": "exec", - "output_limit": 2048, - "timeout": 8 + "scope": "worktree", + "observations": { + "ready": { + "label": "就绪状态", + "purpose": "runtime-readiness", + "probe": { + "argv": [ + "pdm", + "run", + "dev:database", + "probe", + "${dev.instance}" + ], + "cwd": ".", + "kind": "exec", + "output_limit": 2048, + "timeout": 8 + } + } }, - "provision": { - "argv": [ - "pdm", - "run", - "dev:database", - "ensure", - "${dev.instance}" - ], - "cwd": ".", - "env": { - "INKCRE_DATABASE_PROVIDER": "local" + "actions": { + "ensure": { + "argv": [ + "pdm", + "run", + "dev:database", + "ensure", + "${dev.instance}" + ], + "cwd": ".", + "env": { + "INKCRE_DATABASE_PROVIDER": "local" + }, + "kind": "exec", + "intent": "准备database开发资源", + "traits": [ + "runtime", + "provision" + ], + "effect_scope": "当前工作树的database开发资源", + "data_effect": "modify", + "completion": "exit", + "interruption": "wait", + "conditions": [ + "ready" + ], + "completion_timeout": 30, + "poll_interval": 1, + "convergent": true }, - "kind": "exec", - "mode": "run" - }, - "readiness_timeout": 600, - "scope": "worktree", - "stop": { - "argv": [ - "pdm", - "run", - "dev:database", - "stop", - "${dev.instance}" - ], - "cwd": ".", - "kind": "exec", - "timeout": 300 + "stop": { + "argv": [ + "pdm", + "run", + "dev:database", + "stop", + "${dev.instance}" + ], + "cwd": ".", + "kind": "exec", + "intent": "移除database开发资源", + "traits": [ + "runtime", + "provision" + ], + "effect_scope": "当前工作树的database开发资源", + "data_effect": "delete", + "completion": "exit", + "interruption": "wait" + } } } } diff --git a/tests/extensions/bilibili/test_lexical_integration.py b/tests/extensions/bilibili/test_lexical_integration.py new file mode 100644 index 00000000..bc2640ca --- /dev/null +++ b/tests/extensions/bilibili/test_lexical_integration.py @@ -0,0 +1,51 @@ +"""在明确选择的开发数据库验证既有词法维护与检索，无源站访问。""" + +import os +from uuid import uuid4 + +import pytest + +from extensions.bilibili.resolver import VideoResolver +from extensions.bilibili.schema import Video +from inkcre_core.business.info_base.services import BlockService +from inkcre_core.business.lexical_retrieval import LexicalRetrievalManager +from inkcre_core.schemas.info_base.block import BlockForm +from inkcre_core.schemas.lexical_retrieval import LexicalMaintenanceOptions + + +pytestmark = pytest.mark.skipif( + not os.getenv("INKCRE_TEST_DATABASE_URL"), + reason="requires an explicitly selected disposable PostgreSQL runtime", +) + + +def test_existing_maintenance_indexes_saved_bilibili_text(async_runner): + clue = f"bilibili-reader-{uuid4().hex}" + content = Video( + aid="901", + title="事务边界研究", + description=f"这段已保存的简介包含唯一线索 {clue}。", + ) + created = async_runner.run( + BlockService.create( + BlockForm( + resolver=VideoResolver.__rsotype__, + content=content.model_dump_json(exclude_none=True), + ) + ) + ) + assert created.id is not None + try: + report = async_runner.run( + LexicalRetrievalManager.maintain( + LexicalMaintenanceOptions(max_records=100_000, scan_page_size=100) + ) + ) + assert report.indexed >= 1 + result = async_runner.run(LexicalRetrievalManager.retrieve_local(clue)) + match = next(value for value in result.matches if value.block.id == created.id) + assert match.label == content.title + assert clue in match.excerpt + assert match.evidence == "text_substring" + finally: + async_runner.run(BlockService.delete(created.id)) diff --git a/tests/extensions/bilibili/test_reader.py b/tests/extensions/bilibili/test_reader.py new file mode 100644 index 00000000..7970b2b7 --- /dev/null +++ b/tests/extensions/bilibili/test_reader.py @@ -0,0 +1,129 @@ +"""同一 v1 图不依赖采集浏览器即可进入现有词法投影。""" + +import datetime +import json + +import pydantic +import pytest + +from extensions.bilibili.resolver import ( + FolderResolver, + MembershipResolver, + UserResolver, + VideoResolver, +) +from extensions.bilibili.schema import ( + FOLDER_RESOLVER, + MEMBERSHIP_RESOLVER, + USER_RESOLVER, + VIDEO_RESOLVER, +) +from inkcre_core.business.info_base.resolver import ResolverManager, UnknownResolverError +from inkcre_core.business.lexical_retrieval.main import LexicalRetrievalManager +from inkcre_core.schemas.info_base.block import BlockModel + + +def block(resolver: str, content: dict) -> BlockModel: + now = datetime.datetime.now(datetime.UTC) + return BlockModel( + id=41, + resolver=resolver, + content=json.dumps(content, ensure_ascii=False), + storage=None, + created_at=now, + updated_at=now, + ) + + +@pytest.mark.parametrize( + "resolver_cls,resolver_id,content,expected", + [ + ( + UserResolver, + USER_RESOLVER, + {"mid": "100", "name": "测试 UP", "avatar_url": "https://example.test/avatar.jpg"}, + ("测试 UP", "测试 UP"), + ), + ( + FolderResolver, + FOLDER_RESOLVER, + {"id": "90001", "title": "系统设计", "description": "架构资料", "is_private": True}, + ("系统设计", "系统设计\n架构资料"), + ), + ( + VideoResolver, + VIDEO_RESOLVER, + { + "aid": "200", + "bvid": "BV1ab411c7AB", + "title": "数据库事务", + "description": "讨论隔离级别与原子提交。", + "duration_seconds": 180, + "published_at": "2026-01-01T00:00:00Z", + "cover_url": "https://example.test/cover.jpg", + }, + ("数据库事务", "数据库事务\n讨论隔离级别与原子提交。"), + ), + ( + MembershipResolver, + MEMBERSHIP_RESOLVER, + { + "availability": "unavailable", + "page_number": 2, + "position_in_page": 3, + "favored_at": "2026-01-02T00:00:00Z", + "observed_at": "2026-01-03T00:00:00Z", + }, + ("已失效的收藏条目", "已失效的收藏条目"), + ), + ], +) +def test_web_records_project_without_browser( + async_runner, resolver_cls, resolver_id, content, expected +): + label, text = expected + value = block(resolver_id, content) + reader = resolver_cls(value) + assert async_runner.run(reader.get_label()) == label + assert async_runner.run(reader.get_text(context="lexical")) == text + projection = async_runner.run(LexicalRetrievalManager._project(value)) + assert (projection.block, projection.label, projection.text) == (41, label, text) + + +@pytest.mark.parametrize( + "resolver_id,content,expected", + [ + (USER_RESOLVER, {"mid": "100"}, "UID 100"), + (FOLDER_RESOLVER, {"id": "90001"}, "收藏夹 90001"), + (VIDEO_RESOLVER, {"aid": "200", "bvid": "BV1ab411c7AB"}, "BV1ab411c7AB"), + (VIDEO_RESOLVER, {"aid": "200"}, "av200"), + ], +) +def test_sparse_identity_has_a_readable_fallback( + async_runner, resolver_id, content, expected +): + projection = async_runner.run( + LexicalRetrievalManager._project(block(resolver_id, content)) + ) + assert projection.label == projection.text == expected + + +@pytest.mark.parametrize( + "content", + [ + {"aid": 200}, + {"aid": "2١"}, + {"aid": "200", "title": None}, + {"aid": "200", "duration_seconds": "180"}, + {"aid": "200", "published_at": "2026-01-01T00:00:00+08:00"}, + {"aid": "200", "unexpected": "cannot silently reinterpret a future record"}, + ], +) +def test_malformed_v1_records_fail_instead_of_inventing_text(async_runner, content): + with pytest.raises(pydantic.ValidationError): + async_runner.run(VideoResolver(block(VIDEO_RESOLVER, content)).get_text()) + + +def test_unknown_version_is_not_guessed(): + with pytest.raises(UnknownResolverError): + ResolverManager.get(block("extensions.bilibili.video.v2", {"aid": "200"})) diff --git a/tests/migrations/test_source_batch_persistence.py b/tests/migrations/test_source_batch_persistence.py new file mode 100644 index 00000000..9b42a633 --- /dev/null +++ b/tests/migrations/test_source_batch_persistence.py @@ -0,0 +1,475 @@ +"""Database protocol proof for atomic native identity graph batches.""" + +from concurrent.futures import ThreadPoolExecutor +import asyncio +import json +import os +from threading import Barrier +from typing import TypeVar +from uuid import uuid4 + +import psycopg +from psycopg.types.json import Jsonb +import pytest + + +RowT = TypeVar("RowT") + + +def fetch_one(cursor: psycopg.Cursor[RowT]) -> RowT: + row = cursor.fetchone() + assert row is not None, "The query must return a row" + return row + + +def connect(): + url = os.environ["INKCRE_TEST_DATABASE_URL"].replace( + "postgresql+psycopg://", "postgresql://" + ) + return psycopg.connect(url) + + +@pytest.fixture +def scope(): + if not os.getenv("INKCRE_TEST_DATABASE_URL"): + pytest.skip("requires an explicitly selected disposable migrated PostgreSQL runtime") + namespace = f"test/source-batch/{uuid4()}" + source_ids = [] + with connect() as conn: + conn.execute( + "INSERT INTO inkcre.sources_types (id, description, config_schema) " + "VALUES (%s, 'batch test', '{}')", + (namespace,), + ) + for _ in range(2): + source_ids.append( + fetch_one( + conn.execute( + "INSERT INTO inkcre.sources (type, config, state) " + "VALUES (%s, '{}', '{}') RETURNING id", + (namespace,), + ) + )[0] + ) + yield namespace, source_ids + with connect() as conn: + anchors = [ + row[0] + for row in conn.execute( + "SELECT block FROM inkcre.sources WHERE id=ANY(%s) AND block IS NOT NULL", + (source_ids,), + ).fetchall() + ] + conn.execute("DELETE FROM inkcre.sources WHERE id = ANY(%s)", (source_ids,)) + conn.execute("DELETE FROM inkcre.sources_types WHERE id = %s", (namespace,)) + conn.execute( + "DELETE FROM inkcre.blocks WHERE id IN " + "(SELECT block FROM inkcre.collection_records WHERE namespace = %s)", + (namespace,), + ) + conn.execute("DELETE FROM inkcre.blocks WHERE id=ANY(%s)", (anchors,)) + + +def record(key="video:1", **content): + return { + "key": key, + "resolver": "tests.collection.video.v1", + "content": {"id": "1", **content}, + } + + +def commit( # noqa: PLR0913 — the helper mirrors the admitted RPC inputs. + conn, + namespace, + source_id, + records, + *, + expected=None, + next_state=None, + config=None, + relations=None, + root_key=None, +): + sql = "SELECT inkcre.commit_source_batch(%s, %s, %s, %s, %s, %s, %s)" + args = ( + source_id, + namespace, + Jsonb(config or {}), + Jsonb(expected or {}), + Jsonb(next_state or {}), + Jsonb(records), + Jsonb(relations or []), + ) + if root_key is not None: + sql = "SELECT inkcre.commit_source_batch(%s, %s, %s, %s, %s, %s, %s, %s)" + args = (*args, root_key) + return fetch_one( + conn.execute( + sql, + args, + ) + )[0] + + +def test_native_identity_merge_membership_and_atomic_state(scope): + namespace, (first, second) = scope + relations = [{"from_key": "folder:1", "to_key": "video:1", "content": "contains"}] + with connect() as conn: + result = commit( + conn, + namespace, + first, + [record(title="known", description="retain"), record("folder:1", name="A")], + next_state={"page": 1}, + relations=relations, + ) + video = result["blocks"]["video:1"] + folder = result["blocks"]["folder:1"] + conn.execute( + "INSERT INTO inkcre.relations (from_, to_, content) " + "VALUES (%s, %s, 'user:organized')", + (video, folder), + ) + with connect() as conn: + again = commit( + conn, + namespace, + second, + [record(title="updated")], + next_state={"page": 2}, + relations=relations * 2, + ) + assert again["blocks"]["video:1"] == video + assert again["state"] == {"page": 2} + assert json.loads( + fetch_one(conn.execute("SELECT content FROM inkcre.blocks WHERE id = %s", (video,)))[ + 0 + ] + ) == {"id": "1", "title": "updated", "description": "retain"} + assert fetch_one( + conn.execute("SELECT state FROM inkcre.sources WHERE id = %s", (second,)) + )[0] == {"page": 2} + assert ( + fetch_one( + conn.execute( + "SELECT count(*) FROM inkcre.relations " + "WHERE from_=%s AND to_=%s AND content='contains'", + (folder, video), + ) + )[0] + == 1 + ) + assert ( + fetch_one( + conn.execute( + "SELECT count(*) FROM inkcre.relations " + "WHERE from_=%s AND content='user:organized'", + (video,), + ) + )[0] + == 1 + ) + + +def test_root_pointer_commits_with_graph_state_and_unknown_root_rolls_back(scope): + namespace, (first, second) = scope + with connect() as conn: + result = commit( + conn, + namespace, + first, + [record("root")], + next_state={"page": 1}, + root_key="root", + ) + root = result["blocks"]["root"] + first_anchor = result["source_block"] + assert first_anchor != root + assert fetch_one( + conn.execute("SELECT block,state FROM inkcre.sources WHERE id=%s", (first,)) + ) == (first_anchor, {"page": 1}) + second_result = commit(conn, namespace, second, [], root_key="root") + second_anchor = second_result["source_block"] + assert second_anchor not in {root, first_anchor} + assert ( + fetch_one(conn.execute("SELECT block FROM inkcre.sources WHERE id=%s", (second,)))[0] + == second_anchor + ) + assert ( + fetch_one( + conn.execute( + "SELECT count(*) FROM inkcre.relations WHERE from_=ANY(%s) " + "AND to_=%s AND content='collects'", + ([first_anchor, second_anchor], root), + ) + )[0] + == 2 + ) + + async def rename_source(): + from inkcre_core import Core + from inkcre_core.business.source import SourceManager + from inkcre_core.schemas.source import SourceUpdateForm + + core = Core(os.environ["INKCRE_TEST_DATABASE_URL"]) + try: + with core.bind(): + await SourceManager.update(first, SourceUpdateForm(nickname="renamed")) + finally: + await core.aclose() + + asyncio.run(rename_source()) + with connect() as conn: + anchor = fetch_one( + conn.execute( + "SELECT resolver,content FROM inkcre.blocks WHERE id=%s", (first_anchor,) + ) + ) + assert anchor[0] == "core.source.v1" + assert json.loads(anchor[1]) == {"id": first, "type": namespace, "nickname": "renamed"} + assert ( + fetch_one(conn.execute("SELECT resolver FROM inkcre.blocks WHERE id=%s", (root,)))[0] + == "tests.collection.video.v1" + ) + with connect() as conn: + with pytest.raises(psycopg.Error) as caught: + with conn.transaction(): + commit( + conn, + namespace, + first, + [record("new")], + expected={"page": 1}, + next_state={"page": 2}, + root_key="absent", + ) + assert caught.value.sqlstate == "22023" + assert fetch_one( + conn.execute("SELECT block,state FROM inkcre.sources WHERE id=%s", (first,)) + ) == (first_anchor, {"page": 1}) + assert ( + fetch_one( + conn.execute( + "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) + ) + )[0] + == 1 + ) + # Seven positional arguments still resolve to the one eight-argument function. + commit(conn, namespace, first, [], expected={"page": 1}, next_state={"page": 2}) + assert fetch_one( + conn.execute("SELECT block,state FROM inkcre.sources WHERE id=%s", (first,)) + ) == (first_anchor, {"page": 2}) + + +def test_incompatible_source_anchor_never_rewrites_business_block(scope): + namespace, (source_id, _) = scope + with connect() as conn: + result = commit(conn, namespace, source_id, [record(title="business")]) + video = result["blocks"]["video:1"] + conn.execute("UPDATE inkcre.sources SET block=%s WHERE id=%s", (video, source_id)) + with connect() as conn: + with pytest.raises(psycopg.Error) as caught: + with conn.transaction(): + commit( + conn, + namespace, + source_id, + [record(title="must roll back")], + root_key="video:1", + next_state={"page": 1}, + ) + assert caught.value.sqlstate == "22023" + stored = fetch_one( + conn.execute("SELECT resolver,content FROM inkcre.blocks WHERE id=%s", (video,)) + ) + assert stored[0] == "tests.collection.video.v1" + assert json.loads(stored[1])["title"] == "business" + assert ( + fetch_one(conn.execute("SELECT state FROM inkcre.sources WHERE id=%s", (source_id,)))[ + 0 + ] + == {} + ) + + +@pytest.mark.parametrize("failure", ["state", "config", "resolver", "endpoint"]) +def test_conflict_or_invalid_graph_rolls_back_entire_batch(scope, failure): + namespace, (source_id, _) = scope + with connect() as conn: + initial = commit( + conn, namespace, source_id, [record(title="before")], next_state={"page": 1} + ) + with connect() as conn: + with pytest.raises(psycopg.Error) as caught: + with conn.transaction(): + changed = record(title="wrong") + if failure == "resolver": + changed["resolver"] = "tests.collection.incompatible.v1" + commit( + conn, + namespace, + source_id, + [record("video:new"), changed], + expected={} if failure == "state" else {"page": 1}, + config={"changed": True} if failure == "config" else {}, + next_state={"page": 2}, + relations=[{"from_key": "video:1", "to_key": "absent", "content": "contains"}] + if failure == "endpoint" + else [], + ) + assert caught.value.sqlstate == ("PT409" if failure in {"state", "config"} else "22023") + assert ( + fetch_one( + conn.execute( + "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) + ) + )[0] + == 1 + ) + assert fetch_one( + conn.execute("SELECT state FROM inkcre.sources WHERE id=%s", (source_id,)) + )[0] == {"page": 1} + assert ( + json.loads( + fetch_one( + conn.execute( + "SELECT content FROM inkcre.blocks WHERE id=%s", (initial["blocks"]["video:1"],) + ) + )[0] + )["title"] + == "before" + ) + + +@pytest.mark.parametrize( + "records", + [ + [record(), record()], + [record("")], + [{"key": "bad", "resolver": "x", "content": []}], + [record()] * 501, + ], +) +def test_rejects_invalid_or_oversized_records(scope, records): + namespace, (source_id, _) = scope + with connect() as conn: + with pytest.raises(psycopg.Error) as caught: + with conn.transaction(): + commit(conn, namespace, source_id, records) + assert caught.value.sqlstate == "22023" + assert ( + fetch_one( + conn.execute( + "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) + ) + )[0] + == 0 + ) + + +def test_concurrent_sources_reuse_native_identity(scope): + namespace, source_ids = scope + barrier = Barrier(2) + + def execute(source_id): + with connect() as conn: + barrier.wait(timeout=10) + return commit( + conn, namespace, source_id, [record(title="same")], next_state={"done": True} + )["blocks"]["video:1"] + + with ThreadPoolExecutor(max_workers=2) as pool: + ids = list(pool.map(execute, source_ids)) + assert ids[0] == ids[1] + with connect() as conn: + assert ( + fetch_one( + conn.execute( + "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) + ) + )[0] + == 1 + ) + + +def test_rpc_is_invoker_and_does_not_grant_public_access(scope): + with connect() as conn: + row = fetch_one( + conn.execute( + "SELECT p.prosecdef, p.proconfig, " + "EXISTS (SELECT 1 FROM aclexplode(p.proacl) acl " + "WHERE acl.grantee=0 AND acl.privilege_type='EXECUTE') " + "FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace " + "WHERE n.nspname='inkcre' AND p.proname='commit_source_batch'" + ) + ) + assert row == (False, ["search_path=pg_catalog"], False) + + +def test_authenticated_executes_with_ordinary_permissions_and_anonymous_cannot(scope): + namespace, source_ids = scope + with connect() as conn: + if ( + fetch_one( + conn.execute( + "SELECT count(*) FROM pg_roles WHERE rolname IN ('authenticated', 'anonymous')" + ) + )[0] + != 2 + ): + pytest.skip("requires database lifecycle principals in the disposable cluster") + # These grants are local to the disposable test database, not cluster role changes. + conn.execute("GRANT USAGE ON SCHEMA inkcre TO authenticated") + conn.execute( + "GRANT SELECT,INSERT,UPDATE ON inkcre.sources, inkcre.blocks, " + "inkcre.relations, inkcre.collection_records TO authenticated" + ) + conn.execute("GRANT SELECT ON inkcre.storages, inkcre.storage_types TO authenticated") + conn.execute("GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA inkcre TO authenticated") + conn.execute( + "GRANT EXECUTE ON FUNCTION inkcre.commit_source_batch" + "(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb,text) TO authenticated" + ) + conn.execute("SET LOCAL ROLE authenticated") + result = commit(conn, namespace, source_ids[0], [record()], next_state={"ok": True}) + assert result["blocks"]["video:1"] > 0 + with connect() as conn: + with pytest.raises(psycopg.errors.InsufficientPrivilege): + with conn.transaction(): + conn.execute("SET LOCAL ROLE anonymous") + commit(conn, namespace, source_ids[1], [record("denied")]) + assert ( + fetch_one( + conn.execute( + "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) + ) + )[0] + == 1 + ) + + +def test_protocol_projection_declares_the_same_rpc_and_native_mapping(): + from app.database_contract.protocol import ( + protocol_database_function_signatures, + protocol_document, + ) + + relations = protocol_document()["relations"] + assert isinstance(relations, dict) + assert "collection_records" in relations + names, types, returns, returns_set, volatility = protocol_database_function_signatures()[ + "commit_source_batch" + ] + assert names == ( + "source_id", + "namespace", + "expected_config", + "expected_state", + "next_state", + "records", + "relations", + "root_key", + ) + assert types == ("integer", "text", "jsonb", "jsonb", "jsonb", "jsonb", "jsonb", "text") + assert (returns, returns_set, volatility) == ("jsonb", False, "v") diff --git a/tests/test_dev_database_provider.py b/tests/test_dev_database_provider.py new file mode 100644 index 00000000..e2881431 --- /dev/null +++ b/tests/test_dev_database_provider.py @@ -0,0 +1,55 @@ +import json +from io import BytesIO +import tarfile + +from scripts import dev_database_provider as provider + + +def test_schema5_machine_override_selects_the_declared_ssh_provider(tmp_path, monkeypatch): + base = tmp_path / "svc.json" + local = tmp_path / "svc.local.json" + base.write_text(json.dumps({"schema_version": 5, "dev": {"resources": {}}})) + local.write_text( + json.dumps( + { + "schema_version": 5, + "dev": { + "resources": { + "database": { + "actions": { + "ensure": { + "env": { + "INKCRE_DATABASE_PROVIDER": "ssh", + "INKCRE_DATABASE_SSH_TARGET": "test-host", + "INKCRE_DATABASE_SSH_DOCKER_BIN": "/usr/bin/docker", + } + } + } + } + } + }, + } + ) + ) + monkeypatch.setattr(provider, "BASE_CONFIG", base) + monkeypatch.setattr(provider, "LOCAL_CONFIG", local) + for name in ( + "INKCRE_DATABASE_PROVIDER", + "INKCRE_DATABASE_SSH_TARGET", + "INKCRE_DATABASE_SSH_DOCKER_BIN", + ): + monkeypatch.delenv(name, raising=False) + actual = provider.resolve_database_provider() + assert actual.kind == "ssh" + assert actual.target == "test-host" + assert actual.docker_bin == "/usr/bin/docker" + + +def test_remote_build_context_carries_the_sdk_and_required_license(tmp_path): + environment = tmp_path / "compose.env" + environment.write_text("INKCRE_COMPOSE_PROJECT_NAME='test-context'\n") + payload = provider._remote_payload(environment, ("__provider-check__",)) + with tarfile.open(fileobj=BytesIO(payload)) as archive: + names = set(archive.getnames()) + assert "context/inkcre_core/__init__.py" in names + assert "context/LICENSE" in names From 46c2f3f2c35cf67a16003286fd45863d89fff703 Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 00:10:04 +0800 Subject: [PATCH 2/6] =?UTF-8?q?docs(shared):=20=E5=90=8C=E6=AD=A5=20Source?= =?UTF-8?q?=20=E6=89=B9=E6=AC=A1=E4=B8=8E=20B=E7=AB=99=E6=94=B6=E8=97=8F?= =?UTF-8?q?=E5=A5=91=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 固定已经推送到 Hub 的 76dd7c6 - 与 Python 业务实现提交分开维护共享文档引用 --- docs/_shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_shared b/docs/_shared index 17fc19e7..76dd7c65 160000 --- a/docs/_shared +++ b/docs/_shared @@ -1 +1 @@ -Subproject commit 17fc19e7230187a7a989815e7b8303bbb380f478 +Subproject commit 76dd7c655ba7dcc1aeb295dac08ada194f8d662a From e2db59f17f72d29281b57e70dd467cdafd709f1e Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 00:13:48 +0800 Subject: [PATCH 3/6] =?UTF-8?q?fix(bilibili):=20=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E6=B5=8B=E8=AF=95=E4=B8=8E=E6=97=A2=E6=9C=89?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E9=87=8D=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 使用插件专属测试文件名，保留所有读取断言 - 完整仓库检查通过，避免 pytest 同批收集时的模块冲突 --- .../bilibili/{test_reader.py => test_bilibili_reader.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/extensions/bilibili/{test_reader.py => test_bilibili_reader.py} (100%) diff --git a/tests/extensions/bilibili/test_reader.py b/tests/extensions/bilibili/test_bilibili_reader.py similarity index 100% rename from tests/extensions/bilibili/test_reader.py rename to tests/extensions/bilibili/test_bilibili_reader.py From abf89b5c1ebcc6ce97968fd31cba27608902b29d Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 02:08:20 +0800 Subject: [PATCH 4/6] =?UTF-8?q?docs(shared):=20=E6=9B=B4=E6=96=B0=E6=9C=80?= =?UTF-8?q?=E5=B0=8F=E6=94=B6=E8=97=8F=E6=8C=81=E4=B9=85=E5=8C=96=E5=A5=91?= =?UTF-8?q?=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/_shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_shared b/docs/_shared index 76dd7c65..b50921b1 160000 --- a/docs/_shared +++ b/docs/_shared @@ -1 +1 @@ -Subproject commit 76dd7c655ba7dcc1aeb295dac08ada194f8d662a +Subproject commit b50921b19cb6cf81ad4c09786423195974d901b2 From 1eaab34c1ebce157ee42c2ba607a6c5e17b5b34d Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 02:08:57 +0800 Subject: [PATCH 5/6] =?UTF-8?q?fix(collection):=20=E6=92=A4=E5=9B=9E?= =?UTF-8?q?=E9=80=9A=E7=94=A8=E8=BA=AB=E4=BB=BD=E6=98=A0=E5=B0=84=E4=B8=8E?= =?UTF-8?q?=E6=89=B9=E6=AC=A1=20RPC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 删除未发布的辅助表、三次迁移和协议修订，沿用既有信息图 - 保留 B 站读取分发与已观察到的镜像构建修复，撤回 SVC 配置迁移 --- .changes/browser-collection-runtime.fixed.md | 1 + .changes/source-batch.added.md | 1 - app/database_contract/protocol.py | 22 - docs/30-unit-tdd/bilibili-extension.md | 2 +- .../business-pipeline-and-authority.md | 20 - docs/40-deployment/development-environment.md | 4 +- extensions/bilibili/README.md | 2 +- inkcre_core/database_contract/constants.py | 3 +- inkcre_core/schemas/__init__.py | 2 - .../schemas/source/collection_record.py | 31 -- migrations/revision-integrity.json | 3 - ...b2c3d4e5a6_add_source_batch_persistence.py | 187 ------- .../f1b2c3d4e5a7_commit_source_batch_root.py | 303 ----------- .../f1b2c3d4e5a8_preserve_source_anchor.py | 334 ------------ scripts/dev_database_provider.py | 11 - svc.json | 112 ++--- .../test_source_batch_persistence.py | 475 ------------------ tests/test_dev_database_provider.py | 62 +-- 18 files changed, 68 insertions(+), 1507 deletions(-) create mode 100644 .changes/browser-collection-runtime.fixed.md delete mode 100644 .changes/source-batch.added.md delete mode 100644 inkcre_core/schemas/source/collection_record.py delete mode 100644 migrations/versions/f1b2c3d4e5a6_add_source_batch_persistence.py delete mode 100644 migrations/versions/f1b2c3d4e5a7_commit_source_batch_root.py delete mode 100644 migrations/versions/f1b2c3d4e5a8_preserve_source_anchor.py delete mode 100644 tests/migrations/test_source_batch_persistence.py diff --git a/.changes/browser-collection-runtime.fixed.md b/.changes/browser-collection-runtime.fixed.md new file mode 100644 index 00000000..0a457614 --- /dev/null +++ b/.changes/browser-collection-runtime.fixed.md @@ -0,0 +1 @@ +修复远端开发镜像缺少 SDK 源码与 LICENSE 的构建输入，固定 PDM 安装器的兼容依赖；构建失败时保留两个输出流中的原因并遮盖凭据。 diff --git a/.changes/source-batch.added.md b/.changes/source-batch.added.md deleted file mode 100644 index 869ca02e..00000000 --- a/.changes/source-batch.added.md +++ /dev/null @@ -1 +0,0 @@ -数据库协议新增按原生身份幂等保存 Source 批次的能力；信息图、Source 进度与可选根入口原子提交，支持独立 Web Peer 的采集执行。 diff --git a/app/database_contract/protocol.py b/app/database_contract/protocol.py index bf1b37fd..1b20d734 100644 --- a/app/database_contract/protocol.py +++ b/app/database_contract/protocol.py @@ -18,26 +18,6 @@ POSTGREST_OCTET_STREAM_TYPE = f'{PROTOCOL_SCHEMA}."application/octet-stream"' PROTOCOL_FUNCTIONS: dict[str, dict[str, object]] = { - "commit_source_batch": { - "arguments": [ - {"name": "source_id", "type": {"kind": "number", "format": "integer"}}, - {"name": "namespace", "type": {"kind": "string"}}, - *[ - {"name": name, "type": {"kind": "json"}} - for name in ( - "expected_config", - "expected_state", - "next_state", - "records", - "relations", - ) - ], - {"name": "root_key", "type": {"kind": "string"}, "has_default": True}, - ], - "returns": {"kind": "json"}, - "returns_set": False, - "volatility": "volatile", - }, "create_storage_blob": { "arguments": [ { @@ -96,8 +76,6 @@ def _database_type_name(type_document: dict[str, object]) -> str: if isinstance(format_, str): return format_ kind = type_document.get("kind") - if kind == "json": - return "jsonb" if kind == "string": return "text" if kind == "boolean": diff --git a/docs/30-unit-tdd/bilibili-extension.md b/docs/30-unit-tdd/bilibili-extension.md index a37f5482..b7a4cffe 100644 --- a/docs/30-unit-tdd/bilibili-extension.md +++ b/docs/30-unit-tdd/bilibili-extension.md @@ -4,7 +4,7 @@ ## 读取与索引边界 -`extensions/bilibili/schema.py` 验证四类 v1 原生记录，拒绝未知字段、显式 null、非字符串身份和错误类型。原生身份映射由 `collection_records` 拥有；记录之间的关联仅在 Relations，不在 Python 另存引用或内容副本。 +`extensions/bilibili/schema.py` 验证四类 v1 原生记录，拒绝未知字段、显式 null、非字符串身份和错误类型。User、Folder、Video 的原生身份保存在 Block 内容中；Membership 由所属 Folder 和指向 Video 的关系定位，不复制关联 ID。Python 解释这些已保存的资料。 Resolver 只从已持久化内容产生 Block-local 标签与文本。视频与收藏夹投影包括已保存的标题、简介，未知标题回退到 aid、BVID 或列表 ID。词法维护使用既有 `LexicalRetrievalManager` 和普通维护 Job；插件不直接写派生检索表，也不递归抓取相关视频以补全文本。 diff --git a/docs/30-unit-tdd/business-pipeline-and-authority.md b/docs/30-unit-tdd/business-pipeline-and-authority.md index c0dea070..ea1b7778 100644 --- a/docs/30-unit-tdd/business-pipeline-and-authority.md +++ b/docs/30-unit-tdd/business-pipeline-and-authority.md @@ -266,26 +266,6 @@ HTTP 422。普通 REST 的请求输入、错误位置和内容表示见 [rest-in - Agent definition 是 shared database fact；Thread state、bound Tool handlers 与正在执行的 Turn 是 peer-local runtime fact。Agent definition 更新不会改写已经创建的 Thread snapshot。 -## Source 批次持久化 - -浏览器等非 Python 执行者通过数据库协议的 `commit_source_batch` 提交原生记录、稳定关系和 Source 进度。 -`collection_records` 只拥有 `(namespace, native_key) → Block` 的身份映射；语义内容仍由 Block 和 Relation -拥有，不进入 Source.state。namespace 是采集扩展选择的业务身份域，不是执行 Peer 或某次 Job 的身份。 - -一次调用先锁定 Source 并核对调用者观察到的完整 config/state，再取得 namespace 事务锁，串行化同一身份域 -的跨 Source 写入。记录复用既有 Block，顶层合并 JSON 对象，缺省字段保留；resolver 不一致、非 inline -内容或不合法端点拒绝整批。关系按精确端点和稳定 content 去重追加，不清理用户关系或未见条目。 -图与 next_state 在同一事务提交。可选 root_key 指向同 namespace 的业务根；Source.block 仍是该 Source -独占的 `core.source.v1` 锚点，不能直接指向跨 Source 共用的业务实体。事务内建立或刷新锚点的 -`{id,type,nickname}`，并追加锚点到业务根的 `collects` 关系；回包的 source_block 给出实际锚点 ID。 -省略 root_key 保留既有入口，不改写非 Source resolver 的 Block。CAS 冲突为 `PT409`，结果未知时调用者 -重新观察原 Source，不自动重放。 - -namespace 锁将冲突范围保持在扩展的身份域内，代价是同域批次串行；每次最多 500 条记录、2000 条关系和 -4 MiB JSON 输入，不能把无界归档作为单个事务。位置、收藏时间等可变成员事实由扩展的独立成员记录解释， -不借变化的 Relation.content 隐式创建新的关系身份。精确输入、DDL 和准入签名分别由 migration、metadata -与 `app/database_contract/protocol.py` 维护；本节不另建 DTO 权威。 - ## Non-Goals 本文件不承接以下内容： diff --git a/docs/40-deployment/development-environment.md b/docs/40-deployment/development-environment.md index 7c9773f8..7523746f 100644 --- a/docs/40-deployment/development-environment.md +++ b/docs/40-deployment/development-environment.md @@ -12,9 +12,7 @@ Python selector and PDM version. ## Local Startup -The supported complete runtime is an SVC worktree capability. `svc.json` declares schema 5 -resources, actions and readiness observations; use a SVC CLI that accepts this schema. The -database action keeps the worktree identity and delegates provisioning to `scripts/dev_database.py`: +The supported complete runtime is an SVC worktree capability: ```bash svc status . --json diff --git a/extensions/bilibili/README.md b/extensions/bilibili/README.md index 200b0757..0f75044b 100644 --- a/extensions/bilibili/README.md +++ b/extensions/bilibili/README.md @@ -19,7 +19,7 @@ 图关系为 `账号 —owns→ 收藏夹 —contains→ 收藏条目 —item→ 视频 —author→ UP 主`。同一视频可由多个收藏条目引用。成员记录中的页码和页内位置是采集观察事实，不承诺原站的实时绝对顺序。 -本 Distribution 不注册 Source，不下载视频二进制、字幕、封面或其他远程资料，不恢复失效视频，也不修改源站收藏。封面 URL 仅作为已保存事实被解释。原生身份去重与批次状态原子提交由采集端和数据库协议完成。 +本 Distribution 不注册 Source，不下载视频二进制、字幕、封面或其他远程资料，不恢复失效视频，也不修改源站收藏。封面 URL 仅作为已保存事实被解释。采集分发按 B 站原生身份保存普通 Block 与 Relation，Python 读取分发解释已保存的资料。 ## 兼容与验证 diff --git a/inkcre_core/database_contract/constants.py b/inkcre_core/database_contract/constants.py index 80979f4f..beb328a3 100644 --- a/inkcre_core/database_contract/constants.py +++ b/inkcre_core/database_contract/constants.py @@ -4,7 +4,7 @@ CONTRACT_FORMAT = 1 -CONTRACT_REVISION = "source-batch-persistence-v1" +CONTRACT_REVISION = "extension-distribution-open-v1" PROTOCOL_SCHEMA = "inkcre" INTERNAL_SCHEMA = "inkcre_internal" @@ -36,7 +36,6 @@ "block_embeddings", "block_lexical_records", "blocks", - "collection_records", "peers", "configs", "extensions", diff --git a/inkcre_core/schemas/__init__.py b/inkcre_core/schemas/__init__.py index f9e5038d..c7eb0f62 100644 --- a/inkcre_core/schemas/__init__.py +++ b/inkcre_core/schemas/__init__.py @@ -23,7 +23,6 @@ "DeploymentConfigModel", "SinkModel", "SinkTypeModel", - "CollectionRecordModel", ] import sqlalchemy.orm @@ -55,4 +54,3 @@ from .deployment_config import DeploymentConfigModel from .lexical_retrieval import BlockLexicalRecordModel from .sink import SinkModel, SinkTypeModel -from .source.collection_record import CollectionRecordModel diff --git a/inkcre_core/schemas/source/collection_record.py b/inkcre_core/schemas/source/collection_record.py deleted file mode 100644 index b7e116a0..00000000 --- a/inkcre_core/schemas/source/collection_record.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Native identity lookup; Block content remains the information authority.""" - -import sqlalchemy -import sqlmodel - - -class CollectionRecordModel(sqlmodel.SQLModel, table=True): - __tablename__ = "collection_records" # type: ignore - __table_args__ = ( - sqlalchemy.CheckConstraint( - "length(namespace) BETWEEN 1 AND 200", name="collection_record_namespace" - ), - sqlalchemy.CheckConstraint( - "length(native_key) BETWEEN 1 AND 1000", name="collection_record_key" - ), - ) - - namespace: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) - ) - native_key: str = sqlmodel.Field( - sa_column=sqlalchemy.Column(sqlalchemy.Text, primary_key=True) - ) - block: int = sqlmodel.Field( - sa_column=sqlalchemy.Column( - sqlalchemy.Integer, - sqlalchemy.ForeignKey("blocks.id", ondelete="CASCADE", onupdate="CASCADE"), - nullable=False, - index=True, - ) - ) diff --git a/migrations/revision-integrity.json b/migrations/revision-integrity.json index 47d2b216..7b0dece0 100644 --- a/migrations/revision-integrity.json +++ b/migrations/revision-integrity.json @@ -28,9 +28,6 @@ "e1f4a5b6c7d8_migrate_memos_attachment_v2.py": "5f3691149e06b28378f79eed7eaacd1c415ee6c92fb83b3b5745a429346d5e01", "e5a01f9e69ef_init.py": "a10de6c0818abc593951a34e51a974ffcd999f60ba5afc441f87ac6cef8d869e", "e6a1b2c3d4f5_extension_peer_participation_v2.py": "bf71d955b1e3e27015f412815d020666157eda6af78ed0472ca2b7d38ada872f", - "f1b2c3d4e5a6_add_source_batch_persistence.py": "f80ebf02ea7475ad2935aabd39904f5062696572847deebd1d1f2f7fd5eea7cd", - "f1b2c3d4e5a7_commit_source_batch_root.py": "0ba844592cecdc1b3847ab1f32f59e3d7f7cafe335f97923ee36a0e6d8f3037e", - "f1b2c3d4e5a8_preserve_source_anchor.py": "0d86aa4f01040a834a8e1dc21fb85c0f60d555115fcaf8b24ba7cbf20d64b6ce", "f2a6c8e4b1d7_add_extension_registry_deployment_state.py": "a2cf3059f87e17292baf9fd10a1acb8b26ccb928845dbe4bec95b374164cd46d", "f2c8a6d1e4b7_add_postgresql_binary_storage.py": "1137323490b324da3f1672cf5a3a8b5edb1e6a258cc10129926bf6e8d232a111", "f7a8b9c0d1e2_add_deployment_configs.py": "9bb58b1d22dccef5817e28df8bcb88ab6edfff70a6823d80b3637752899188fe" diff --git a/migrations/versions/f1b2c3d4e5a6_add_source_batch_persistence.py b/migrations/versions/f1b2c3d4e5a6_add_source_batch_persistence.py deleted file mode 100644 index 01060214..00000000 --- a/migrations/versions/f1b2c3d4e5a6_add_source_batch_persistence.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Add native collection identities and atomic Source graph batches. - -Revision ID: f1b2c3d4e5a6 -Revises: dcf44bc9af73 -""" - -from alembic import op -import sqlalchemy as sa - - -revision = "f1b2c3d4e5a6" -down_revision = "dcf44bc9af73" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.create_table( - "collection_records", - sa.Column("namespace", sa.Text(), nullable=False), - sa.Column("native_key", sa.Text(), nullable=False), - sa.Column("block", sa.Integer(), nullable=False), - sa.CheckConstraint( - "length(namespace) BETWEEN 1 AND 200", name="collection_record_namespace" - ), - sa.CheckConstraint( - "length(native_key) BETWEEN 1 AND 1000", name="collection_record_key" - ), - sa.ForeignKeyConstraint( - ["block"], ["inkcre.blocks.id"], ondelete="CASCADE", onupdate="CASCADE" - ), - sa.PrimaryKeyConstraint("namespace", "native_key"), - schema="inkcre", - ) - op.create_index( - "ix_inkcre_collection_records_block", "collection_records", ["block"], schema="inkcre" - ) - op.execute(""" - CREATE FUNCTION inkcre.commit_source_batch( - source_id integer, - namespace text, - expected_config jsonb, - expected_state jsonb, - next_state jsonb, - records jsonb, - relations jsonb - ) RETURNS jsonb - LANGUAGE plpgsql VOLATILE SECURITY INVOKER - SET search_path = pg_catalog - AS $function$ - DECLARE - source_config jsonb; - source_state jsonb; - item jsonb; - block_row record; - block_id integer; - from_id integer; - to_id integer; - result_blocks jsonb := '{}'::jsonb; - BEGIN - IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 - OR namespace !~ '[^[:space:]]' - OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' - OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(records) IS DISTINCT FROM 'array' - OR jsonb_typeof(relations) IS DISTINCT FROM 'array' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; - END IF; - IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 - OR octet_length((expected_config::text || expected_state::text || next_state::text - || records::text || relations::text)) > 4194304 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; - END IF; - IF EXISTS ( - SELECT 1 FROM jsonb_array_elements(records) AS entry - GROUP BY entry ->> 'key' HAVING count(*) > 1 - ) THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; - END IF; - - SELECT source.config, source.state INTO source_config, source_state - FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; - IF NOT FOUND THEN - RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; - END IF; - IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config - OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state - THEN - RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; - END IF; - - -- Serialize identities across Sources, not just executions of this Source. - PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); - FOR item IN SELECT value FROM jsonb_array_elements(records) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' - OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 - OR (item ->> 'key') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' - OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 - OR (item ->> 'resolver') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; - END IF; - SELECT block.* INTO block_row - FROM inkcre.collection_records AS identity - JOIN inkcre.blocks AS block ON block.id = identity.block - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' - FOR UPDATE OF block; - IF FOUND THEN - IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' - OR block_row.storage IS NOT NULL - OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; - END IF; - block_id := block_row.id; - UPDATE inkcre.blocks AS block - SET content = (block_row.content::jsonb || (item -> 'content'))::text - WHERE block.id = block_id; - ELSE - INSERT INTO inkcre.blocks (storage, resolver, content) - VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) - RETURNING id INTO block_id; - INSERT INTO inkcre.collection_records (namespace, native_key, block) - VALUES ($2, item ->> 'key', block_id); - END IF; - result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); - END LOOP; - - FOR item IN SELECT value FROM jsonb_array_elements(relations) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' - OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; - END IF; - SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; - SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; - IF from_id IS NULL OR to_id IS NULL THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; - END IF; - INSERT INTO inkcre.relations (from_, to_, content) - SELECT from_id, to_id, item ->> 'content' - WHERE NOT EXISTS ( - SELECT 1 FROM inkcre.relations AS relation - WHERE relation.from_ = from_id AND relation.to_ = to_id - AND relation.content = item ->> 'content' - ); - END LOOP; - UPDATE inkcre.sources AS source SET state = next_state WHERE source.id = source_id; - RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state); - END; - $function$; - """) - op.execute( - "REVOKE ALL ON FUNCTION inkcre.commit_source_batch(integer, text, jsonb, jsonb, jsonb, jsonb, jsonb) FROM PUBLIC" - ) - op.execute( - "UPDATE inkcre_internal.contract_state " - "SET contract_revision='source-batch-persistence-v1',updated_at=CURRENT_TIMESTAMP " - "WHERE singleton" - ) - - -def downgrade() -> None: - op.execute( - "DROP FUNCTION inkcre.commit_source_batch(integer, text, jsonb, jsonb, jsonb, jsonb, jsonb)" - ) - op.drop_table("collection_records", schema="inkcre") - op.execute( - "UPDATE inkcre_internal.contract_state " - "SET contract_revision='extension-distribution-open-v1',updated_at=CURRENT_TIMESTAMP " - "WHERE singleton" - ) diff --git a/migrations/versions/f1b2c3d4e5a7_commit_source_batch_root.py b/migrations/versions/f1b2c3d4e5a7_commit_source_batch_root.py deleted file mode 100644 index 652cc73c..00000000 --- a/migrations/versions/f1b2c3d4e5a7_commit_source_batch_root.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Commit the optional Source graph root with its batch. - -Revision ID: f1b2c3d4e5a7 -Revises: f1b2c3d4e5a6 -""" - -from alembic import op - - -revision = "f1b2c3d4e5a7" -down_revision = "f1b2c3d4e5a6" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.execute( - "DROP FUNCTION inkcre.commit_source_batch(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb)" - ) - op.execute(""" - CREATE FUNCTION inkcre.commit_source_batch( - source_id integer, - namespace text, - expected_config jsonb, - expected_state jsonb, - next_state jsonb, - records jsonb, - relations jsonb, - root_key text DEFAULT NULL - ) RETURNS jsonb - LANGUAGE plpgsql VOLATILE SECURITY INVOKER - SET search_path = pg_catalog - AS $function$ - DECLARE - source_config jsonb; - source_state jsonb; - item jsonb; - block_row record; - block_id integer; - from_id integer; - to_id integer; - root_block_id integer; - result_blocks jsonb := '{}'::jsonb; - BEGIN - IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 - OR namespace !~ '[^[:space:]]' - OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' - OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(records) IS DISTINCT FROM 'array' - OR jsonb_typeof(relations) IS DISTINCT FROM 'array' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; - END IF; - IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 - OR octet_length((expected_config::text || expected_state::text || next_state::text - || records::text || relations::text)) > 4194304 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; - END IF; - IF EXISTS ( - SELECT 1 FROM jsonb_array_elements(records) AS entry - GROUP BY entry ->> 'key' HAVING count(*) > 1 - ) THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; - END IF; - - SELECT source.config, source.state INTO source_config, source_state - FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; - IF NOT FOUND THEN - RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; - END IF; - IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config - OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state - THEN - RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; - END IF; - - -- Serialize identities across Sources, not just executions of this Source. - PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); - FOR item IN SELECT value FROM jsonb_array_elements(records) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' - OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 - OR (item ->> 'key') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' - OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 - OR (item ->> 'resolver') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; - END IF; - SELECT block.* INTO block_row - FROM inkcre.collection_records AS identity - JOIN inkcre.blocks AS block ON block.id = identity.block - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' - FOR UPDATE OF block; - IF FOUND THEN - IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' - OR block_row.storage IS NOT NULL - OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; - END IF; - block_id := block_row.id; - UPDATE inkcre.blocks AS block - SET content = (block_row.content::jsonb || (item -> 'content'))::text - WHERE block.id = block_id; - ELSE - INSERT INTO inkcre.blocks (storage, resolver, content) - VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) - RETURNING id INTO block_id; - INSERT INTO inkcre.collection_records (namespace, native_key, block) - VALUES ($2, item ->> 'key', block_id); - END IF; - result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); - END LOOP; - - FOR item IN SELECT value FROM jsonb_array_elements(relations) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' - OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; - END IF; - SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; - SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; - IF from_id IS NULL OR to_id IS NULL THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; - END IF; - INSERT INTO inkcre.relations (from_, to_, content) - SELECT from_id, to_id, item ->> 'content' - WHERE NOT EXISTS ( - SELECT 1 FROM inkcre.relations AS relation - WHERE relation.from_ = from_id AND relation.to_ = to_id - AND relation.content = item ->> 'content' - ); - END LOOP; - IF root_key IS NOT NULL THEN - SELECT identity.block INTO root_block_id - FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = root_key; - IF root_block_id IS NULL THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source root identity is unknown'; - END IF; - END IF; - UPDATE inkcre.sources AS source - SET state = next_state, block = coalesce(root_block_id, source.block) - WHERE source.id = source_id; - RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state); - END; - $function$; - """) - op.execute( - "REVOKE ALL ON FUNCTION inkcre.commit_source_batch(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb,text) FROM PUBLIC" - ) - - -def downgrade() -> None: - op.execute( - "DROP FUNCTION inkcre.commit_source_batch(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb,text)" - ) - op.execute(""" - CREATE FUNCTION inkcre.commit_source_batch( - source_id integer, - namespace text, - expected_config jsonb, - expected_state jsonb, - next_state jsonb, - records jsonb, - relations jsonb - ) RETURNS jsonb - LANGUAGE plpgsql VOLATILE SECURITY INVOKER - SET search_path = pg_catalog - AS $function$ - DECLARE - source_config jsonb; - source_state jsonb; - item jsonb; - block_row record; - block_id integer; - from_id integer; - to_id integer; - result_blocks jsonb := '{}'::jsonb; - BEGIN - IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 - OR namespace !~ '[^[:space:]]' - OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' - OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(records) IS DISTINCT FROM 'array' - OR jsonb_typeof(relations) IS DISTINCT FROM 'array' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; - END IF; - IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 - OR octet_length((expected_config::text || expected_state::text || next_state::text - || records::text || relations::text)) > 4194304 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; - END IF; - IF EXISTS ( - SELECT 1 FROM jsonb_array_elements(records) AS entry - GROUP BY entry ->> 'key' HAVING count(*) > 1 - ) THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; - END IF; - - SELECT source.config, source.state INTO source_config, source_state - FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; - IF NOT FOUND THEN - RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; - END IF; - IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config - OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state - THEN - RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; - END IF; - - -- Serialize identities across Sources, not just executions of this Source. - PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); - FOR item IN SELECT value FROM jsonb_array_elements(records) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' - OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 - OR (item ->> 'key') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' - OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 - OR (item ->> 'resolver') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; - END IF; - SELECT block.* INTO block_row - FROM inkcre.collection_records AS identity - JOIN inkcre.blocks AS block ON block.id = identity.block - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' - FOR UPDATE OF block; - IF FOUND THEN - IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' - OR block_row.storage IS NOT NULL - OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; - END IF; - block_id := block_row.id; - UPDATE inkcre.blocks AS block - SET content = (block_row.content::jsonb || (item -> 'content'))::text - WHERE block.id = block_id; - ELSE - INSERT INTO inkcre.blocks (storage, resolver, content) - VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) - RETURNING id INTO block_id; - INSERT INTO inkcre.collection_records (namespace, native_key, block) - VALUES ($2, item ->> 'key', block_id); - END IF; - result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); - END LOOP; - - FOR item IN SELECT value FROM jsonb_array_elements(relations) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' - OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; - END IF; - SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; - SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; - IF from_id IS NULL OR to_id IS NULL THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; - END IF; - INSERT INTO inkcre.relations (from_, to_, content) - SELECT from_id, to_id, item ->> 'content' - WHERE NOT EXISTS ( - SELECT 1 FROM inkcre.relations AS relation - WHERE relation.from_ = from_id AND relation.to_ = to_id - AND relation.content = item ->> 'content' - ); - END LOOP; - UPDATE inkcre.sources AS source SET state = next_state WHERE source.id = source_id; - RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state); - END; - $function$; - """) - op.execute( - "REVOKE ALL ON FUNCTION inkcre.commit_source_batch(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb) FROM PUBLIC" - ) diff --git a/migrations/versions/f1b2c3d4e5a8_preserve_source_anchor.py b/migrations/versions/f1b2c3d4e5a8_preserve_source_anchor.py deleted file mode 100644 index 56c56dff..00000000 --- a/migrations/versions/f1b2c3d4e5a8_preserve_source_anchor.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Preserve Source anchor identity while linking collected graph roots. - -Revision ID: f1b2c3d4e5a8 -Revises: f1b2c3d4e5a7 -""" - -from alembic import op - - -revision = "f1b2c3d4e5a8" -down_revision = "f1b2c3d4e5a7" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.execute(""" - CREATE OR REPLACE FUNCTION inkcre.commit_source_batch( - source_id integer, - namespace text, - expected_config jsonb, - expected_state jsonb, - next_state jsonb, - records jsonb, - relations jsonb, - root_key text DEFAULT NULL - ) RETURNS jsonb - LANGUAGE plpgsql VOLATILE SECURITY INVOKER - SET search_path = pg_catalog - AS $function$ - DECLARE - source_config jsonb; - source_state jsonb; - source_type text; - source_nickname text; - source_anchor_id integer; - anchor_content jsonb; - anchor_row record; - item jsonb; - block_row record; - block_id integer; - from_id integer; - to_id integer; - root_block_id integer; - result_blocks jsonb := '{}'::jsonb; - BEGIN - IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 - OR namespace !~ '[^[:space:]]' - OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' - OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(records) IS DISTINCT FROM 'array' - OR jsonb_typeof(relations) IS DISTINCT FROM 'array' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; - END IF; - IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 - OR octet_length((expected_config::text || expected_state::text || next_state::text - || records::text || relations::text)) > 4194304 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; - END IF; - IF EXISTS ( - SELECT 1 FROM jsonb_array_elements(records) AS entry - GROUP BY entry ->> 'key' HAVING count(*) > 1 - ) THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; - END IF; - - SELECT source.config, source.state, source.type, source.nickname, source.block - INTO source_config, source_state, source_type, source_nickname, source_anchor_id - FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; - IF NOT FOUND THEN - RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; - END IF; - IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config - OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state - THEN - RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; - END IF; - - -- Serialize identities across Sources, not just executions of this Source. - PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); - FOR item IN SELECT value FROM jsonb_array_elements(records) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' - OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 - OR (item ->> 'key') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' - OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 - OR (item ->> 'resolver') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; - END IF; - SELECT block.* INTO block_row - FROM inkcre.collection_records AS identity - JOIN inkcre.blocks AS block ON block.id = identity.block - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' - FOR UPDATE OF block; - IF FOUND THEN - IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' - OR block_row.storage IS NOT NULL - OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; - END IF; - block_id := block_row.id; - UPDATE inkcre.blocks AS block - SET content = (block_row.content::jsonb || (item -> 'content'))::text - WHERE block.id = block_id; - ELSE - INSERT INTO inkcre.blocks (storage, resolver, content) - VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) - RETURNING id INTO block_id; - INSERT INTO inkcre.collection_records (namespace, native_key, block) - VALUES ($2, item ->> 'key', block_id); - END IF; - result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); - END LOOP; - - FOR item IN SELECT value FROM jsonb_array_elements(relations) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' - OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; - END IF; - SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; - SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; - IF from_id IS NULL OR to_id IS NULL THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; - END IF; - INSERT INTO inkcre.relations (from_, to_, content) - SELECT from_id, to_id, item ->> 'content' - WHERE NOT EXISTS ( - SELECT 1 FROM inkcre.relations AS relation - WHERE relation.from_ = from_id AND relation.to_ = to_id - AND relation.content = item ->> 'content' - ); - END LOOP; - IF root_key IS NOT NULL THEN - SELECT identity.block INTO root_block_id - FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = root_key; - IF root_block_id IS NULL THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source root identity is unknown'; - END IF; - anchor_content := jsonb_build_object('id', source_id, 'type', source_type, 'nickname', source_nickname); - IF source_anchor_id IS NULL THEN - INSERT INTO inkcre.blocks (storage, resolver, content) - VALUES (NULL, 'core.source.v1', anchor_content::text) - RETURNING id INTO source_anchor_id; - ELSE - SELECT block.* INTO anchor_row - FROM inkcre.blocks AS block WHERE block.id = source_anchor_id FOR UPDATE; - IF NOT FOUND OR anchor_row.resolver IS DISTINCT FROM 'core.source.v1' - OR anchor_row.storage IS NOT NULL - OR jsonb_typeof(anchor_row.content::jsonb) IS DISTINCT FROM 'object' - OR (anchor_row.content::jsonb -> 'id') IS DISTINCT FROM to_jsonb(source_id) - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source anchor is incompatible'; - END IF; - UPDATE inkcre.blocks AS block SET content = anchor_content::text - WHERE block.id = source_anchor_id; - END IF; - INSERT INTO inkcre.relations (from_, to_, content) - SELECT source_anchor_id, root_block_id, 'collects' - WHERE NOT EXISTS ( - SELECT 1 FROM inkcre.relations AS relation - WHERE relation.from_ = source_anchor_id AND relation.to_ = root_block_id - AND relation.content = 'collects' - ); - END IF; - UPDATE inkcre.sources AS source - SET state = next_state, block = source_anchor_id - WHERE source.id = source_id; - RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state, 'source_block', source_anchor_id); - END; - $function$; - """) - - -def downgrade() -> None: - op.execute(""" - CREATE OR REPLACE FUNCTION inkcre.commit_source_batch( - source_id integer, - namespace text, - expected_config jsonb, - expected_state jsonb, - next_state jsonb, - records jsonb, - relations jsonb, - root_key text DEFAULT NULL - ) RETURNS jsonb - LANGUAGE plpgsql VOLATILE SECURITY INVOKER - SET search_path = pg_catalog - AS $function$ - DECLARE - source_config jsonb; - source_state jsonb; - item jsonb; - block_row record; - block_id integer; - from_id integer; - to_id integer; - root_block_id integer; - result_blocks jsonb := '{}'::jsonb; - BEGIN - IF namespace IS NULL OR length(namespace) NOT BETWEEN 1 AND 200 - OR namespace !~ '[^[:space:]]' - OR jsonb_typeof(expected_config) IS DISTINCT FROM 'object' - OR jsonb_typeof(expected_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(next_state) IS DISTINCT FROM 'object' - OR jsonb_typeof(records) IS DISTINCT FROM 'array' - OR jsonb_typeof(relations) IS DISTINCT FROM 'array' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source batch shape'; - END IF; - IF jsonb_array_length(records) > 500 OR jsonb_array_length(relations) > 2000 - OR octet_length((expected_config::text || expected_state::text || next_state::text - || records::text || relations::text)) > 4194304 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source batch exceeds size limits'; - END IF; - IF EXISTS ( - SELECT 1 FROM jsonb_array_elements(records) AS entry - GROUP BY entry ->> 'key' HAVING count(*) > 1 - ) THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'duplicate native key in Source batch'; - END IF; - - SELECT source.config, source.state INTO source_config, source_state - FROM inkcre.sources AS source WHERE source.id = source_id FOR UPDATE; - IF NOT FOUND THEN - RAISE EXCEPTION USING ERRCODE = 'P0002', MESSAGE = 'Source does not exist'; - END IF; - IF coalesce(source_config, '{}'::jsonb) IS DISTINCT FROM expected_config - OR coalesce(source_state, '{}'::jsonb) IS DISTINCT FROM expected_state - THEN - RAISE EXCEPTION USING ERRCODE = 'PT409', MESSAGE = 'Source configuration or state changed'; - END IF; - - -- Serialize identities across Sources, not just executions of this Source. - PERFORM pg_advisory_xact_lock(hashtextextended('collection-records:' || namespace, 0)); - FOR item IN SELECT value FROM jsonb_array_elements(records) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['key', 'resolver', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'key') IS DISTINCT FROM 'string' - OR length(item ->> 'key') NOT BETWEEN 1 AND 1000 - OR (item ->> 'key') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'resolver') IS DISTINCT FROM 'string' - OR length(item ->> 'resolver') NOT BETWEEN 1 AND 500 - OR (item ->> 'resolver') !~ '[^[:space:]]' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source record'; - END IF; - SELECT block.* INTO block_row - FROM inkcre.collection_records AS identity - JOIN inkcre.blocks AS block ON block.id = identity.block - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'key' - FOR UPDATE OF block; - IF FOUND THEN - IF block_row.resolver IS DISTINCT FROM item ->> 'resolver' - OR block_row.storage IS NOT NULL - OR jsonb_typeof(block_row.content::jsonb) IS DISTINCT FROM 'object' - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'native identity is bound to an incompatible Block'; - END IF; - block_id := block_row.id; - UPDATE inkcre.blocks AS block - SET content = (block_row.content::jsonb || (item -> 'content'))::text - WHERE block.id = block_id; - ELSE - INSERT INTO inkcre.blocks (storage, resolver, content) - VALUES (NULL, item ->> 'resolver', (item -> 'content')::text) - RETURNING id INTO block_id; - INSERT INTO inkcre.collection_records (namespace, native_key, block) - VALUES ($2, item ->> 'key', block_id); - END IF; - result_blocks := result_blocks || jsonb_build_object(item ->> 'key', block_id); - END LOOP; - - FOR item IN SELECT value FROM jsonb_array_elements(relations) - LOOP - IF jsonb_typeof(item) IS DISTINCT FROM 'object' - OR item - ARRAY['from_key', 'to_key', 'content'] <> '{}'::jsonb - OR jsonb_typeof(item -> 'from_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'to_key') IS DISTINCT FROM 'string' - OR jsonb_typeof(item -> 'content') IS DISTINCT FROM 'string' - OR length(item ->> 'content') NOT BETWEEN 1 AND 1000 - THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'invalid Source relation'; - END IF; - SELECT identity.block INTO from_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'from_key'; - SELECT identity.block INTO to_id FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = item ->> 'to_key'; - IF from_id IS NULL OR to_id IS NULL THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source relation endpoint is unknown'; - END IF; - INSERT INTO inkcre.relations (from_, to_, content) - SELECT from_id, to_id, item ->> 'content' - WHERE NOT EXISTS ( - SELECT 1 FROM inkcre.relations AS relation - WHERE relation.from_ = from_id AND relation.to_ = to_id - AND relation.content = item ->> 'content' - ); - END LOOP; - IF root_key IS NOT NULL THEN - SELECT identity.block INTO root_block_id - FROM inkcre.collection_records AS identity - WHERE identity.namespace = $2 AND identity.native_key = root_key; - IF root_block_id IS NULL THEN - RAISE EXCEPTION USING ERRCODE = '22023', MESSAGE = 'Source root identity is unknown'; - END IF; - END IF; - UPDATE inkcre.sources AS source - SET state = next_state, block = coalesce(root_block_id, source.block) - WHERE source.id = source_id; - RETURN jsonb_build_object('blocks', result_blocks, 'state', next_state); - END; - $function$; - """) diff --git a/scripts/dev_database_provider.py b/scripts/dev_database_provider.py index e52914ef..a65ae0d4 100644 --- a/scripts/dev_database_provider.py +++ b/scripts/dev_database_provider.py @@ -93,17 +93,6 @@ def _provision_environment( legacy_profile: str | None, ) -> dict[str, str]: dev = config.get("dev", {}) - resources = dev.get("resources") - if isinstance(resources, dict): - environment = ( - resources.get("database", {}).get("actions", {}).get("ensure", {}).get("env", {}) - ) - if not isinstance(environment, dict) or not all( - isinstance(name, str) and isinstance(value, str) - for name, value in environment.items() - ): - raise ValueError("database provision environment must contain string values") - return environment targets = dev.get("targets") if targets is None: profile = dev.get("profile") or legacy_profile diff --git a/svc.json b/svc.json index 546f68d8..f652b409 100644 --- a/svc.json +++ b/svc.json @@ -1,78 +1,50 @@ { - "schema_version": 5, + "schema_version": 4, "dev": { - "resources": { + "targets": { "database": { - "scope": "worktree", - "observations": { - "ready": { - "label": "就绪状态", - "purpose": "runtime-readiness", - "probe": { - "argv": [ - "pdm", - "run", - "dev:database", - "probe", - "${dev.instance}" - ], - "cwd": ".", - "kind": "exec", - "output_limit": 2048, - "timeout": 8 - } - } + "poll_interval": 1, + "probe": { + "argv": [ + "pdm", + "run", + "dev:database", + "probe", + "${dev.instance}" + ], + "cwd": ".", + "kind": "exec", + "output_limit": 2048, + "timeout": 8 }, - "actions": { - "ensure": { - "argv": [ - "pdm", - "run", - "dev:database", - "ensure", - "${dev.instance}" - ], - "cwd": ".", - "env": { - "INKCRE_DATABASE_PROVIDER": "local" - }, - "kind": "exec", - "intent": "准备database开发资源", - "traits": [ - "runtime", - "provision" - ], - "effect_scope": "当前工作树的database开发资源", - "data_effect": "modify", - "completion": "exit", - "interruption": "wait", - "conditions": [ - "ready" - ], - "completion_timeout": 30, - "poll_interval": 1, - "convergent": true + "provision": { + "argv": [ + "pdm", + "run", + "dev:database", + "ensure", + "${dev.instance}" + ], + "cwd": ".", + "env": { + "INKCRE_DATABASE_PROVIDER": "local" }, - "stop": { - "argv": [ - "pdm", - "run", - "dev:database", - "stop", - "${dev.instance}" - ], - "cwd": ".", - "kind": "exec", - "intent": "移除database开发资源", - "traits": [ - "runtime", - "provision" - ], - "effect_scope": "当前工作树的database开发资源", - "data_effect": "delete", - "completion": "exit", - "interruption": "wait" - } + "kind": "exec", + "mode": "run" + }, + "readiness_timeout": 600, + "scope": "worktree", + "stop": { + "argv": [ + "pdm", + "run", + "dev:database", + "stop", + "${dev.instance}" + ], + "cwd": ".", + "kind": "exec", + "timeout": 300 } } } diff --git a/tests/migrations/test_source_batch_persistence.py b/tests/migrations/test_source_batch_persistence.py deleted file mode 100644 index 9b42a633..00000000 --- a/tests/migrations/test_source_batch_persistence.py +++ /dev/null @@ -1,475 +0,0 @@ -"""Database protocol proof for atomic native identity graph batches.""" - -from concurrent.futures import ThreadPoolExecutor -import asyncio -import json -import os -from threading import Barrier -from typing import TypeVar -from uuid import uuid4 - -import psycopg -from psycopg.types.json import Jsonb -import pytest - - -RowT = TypeVar("RowT") - - -def fetch_one(cursor: psycopg.Cursor[RowT]) -> RowT: - row = cursor.fetchone() - assert row is not None, "The query must return a row" - return row - - -def connect(): - url = os.environ["INKCRE_TEST_DATABASE_URL"].replace( - "postgresql+psycopg://", "postgresql://" - ) - return psycopg.connect(url) - - -@pytest.fixture -def scope(): - if not os.getenv("INKCRE_TEST_DATABASE_URL"): - pytest.skip("requires an explicitly selected disposable migrated PostgreSQL runtime") - namespace = f"test/source-batch/{uuid4()}" - source_ids = [] - with connect() as conn: - conn.execute( - "INSERT INTO inkcre.sources_types (id, description, config_schema) " - "VALUES (%s, 'batch test', '{}')", - (namespace,), - ) - for _ in range(2): - source_ids.append( - fetch_one( - conn.execute( - "INSERT INTO inkcre.sources (type, config, state) " - "VALUES (%s, '{}', '{}') RETURNING id", - (namespace,), - ) - )[0] - ) - yield namespace, source_ids - with connect() as conn: - anchors = [ - row[0] - for row in conn.execute( - "SELECT block FROM inkcre.sources WHERE id=ANY(%s) AND block IS NOT NULL", - (source_ids,), - ).fetchall() - ] - conn.execute("DELETE FROM inkcre.sources WHERE id = ANY(%s)", (source_ids,)) - conn.execute("DELETE FROM inkcre.sources_types WHERE id = %s", (namespace,)) - conn.execute( - "DELETE FROM inkcre.blocks WHERE id IN " - "(SELECT block FROM inkcre.collection_records WHERE namespace = %s)", - (namespace,), - ) - conn.execute("DELETE FROM inkcre.blocks WHERE id=ANY(%s)", (anchors,)) - - -def record(key="video:1", **content): - return { - "key": key, - "resolver": "tests.collection.video.v1", - "content": {"id": "1", **content}, - } - - -def commit( # noqa: PLR0913 — the helper mirrors the admitted RPC inputs. - conn, - namespace, - source_id, - records, - *, - expected=None, - next_state=None, - config=None, - relations=None, - root_key=None, -): - sql = "SELECT inkcre.commit_source_batch(%s, %s, %s, %s, %s, %s, %s)" - args = ( - source_id, - namespace, - Jsonb(config or {}), - Jsonb(expected or {}), - Jsonb(next_state or {}), - Jsonb(records), - Jsonb(relations or []), - ) - if root_key is not None: - sql = "SELECT inkcre.commit_source_batch(%s, %s, %s, %s, %s, %s, %s, %s)" - args = (*args, root_key) - return fetch_one( - conn.execute( - sql, - args, - ) - )[0] - - -def test_native_identity_merge_membership_and_atomic_state(scope): - namespace, (first, second) = scope - relations = [{"from_key": "folder:1", "to_key": "video:1", "content": "contains"}] - with connect() as conn: - result = commit( - conn, - namespace, - first, - [record(title="known", description="retain"), record("folder:1", name="A")], - next_state={"page": 1}, - relations=relations, - ) - video = result["blocks"]["video:1"] - folder = result["blocks"]["folder:1"] - conn.execute( - "INSERT INTO inkcre.relations (from_, to_, content) " - "VALUES (%s, %s, 'user:organized')", - (video, folder), - ) - with connect() as conn: - again = commit( - conn, - namespace, - second, - [record(title="updated")], - next_state={"page": 2}, - relations=relations * 2, - ) - assert again["blocks"]["video:1"] == video - assert again["state"] == {"page": 2} - assert json.loads( - fetch_one(conn.execute("SELECT content FROM inkcre.blocks WHERE id = %s", (video,)))[ - 0 - ] - ) == {"id": "1", "title": "updated", "description": "retain"} - assert fetch_one( - conn.execute("SELECT state FROM inkcre.sources WHERE id = %s", (second,)) - )[0] == {"page": 2} - assert ( - fetch_one( - conn.execute( - "SELECT count(*) FROM inkcre.relations " - "WHERE from_=%s AND to_=%s AND content='contains'", - (folder, video), - ) - )[0] - == 1 - ) - assert ( - fetch_one( - conn.execute( - "SELECT count(*) FROM inkcre.relations " - "WHERE from_=%s AND content='user:organized'", - (video,), - ) - )[0] - == 1 - ) - - -def test_root_pointer_commits_with_graph_state_and_unknown_root_rolls_back(scope): - namespace, (first, second) = scope - with connect() as conn: - result = commit( - conn, - namespace, - first, - [record("root")], - next_state={"page": 1}, - root_key="root", - ) - root = result["blocks"]["root"] - first_anchor = result["source_block"] - assert first_anchor != root - assert fetch_one( - conn.execute("SELECT block,state FROM inkcre.sources WHERE id=%s", (first,)) - ) == (first_anchor, {"page": 1}) - second_result = commit(conn, namespace, second, [], root_key="root") - second_anchor = second_result["source_block"] - assert second_anchor not in {root, first_anchor} - assert ( - fetch_one(conn.execute("SELECT block FROM inkcre.sources WHERE id=%s", (second,)))[0] - == second_anchor - ) - assert ( - fetch_one( - conn.execute( - "SELECT count(*) FROM inkcre.relations WHERE from_=ANY(%s) " - "AND to_=%s AND content='collects'", - ([first_anchor, second_anchor], root), - ) - )[0] - == 2 - ) - - async def rename_source(): - from inkcre_core import Core - from inkcre_core.business.source import SourceManager - from inkcre_core.schemas.source import SourceUpdateForm - - core = Core(os.environ["INKCRE_TEST_DATABASE_URL"]) - try: - with core.bind(): - await SourceManager.update(first, SourceUpdateForm(nickname="renamed")) - finally: - await core.aclose() - - asyncio.run(rename_source()) - with connect() as conn: - anchor = fetch_one( - conn.execute( - "SELECT resolver,content FROM inkcre.blocks WHERE id=%s", (first_anchor,) - ) - ) - assert anchor[0] == "core.source.v1" - assert json.loads(anchor[1]) == {"id": first, "type": namespace, "nickname": "renamed"} - assert ( - fetch_one(conn.execute("SELECT resolver FROM inkcre.blocks WHERE id=%s", (root,)))[0] - == "tests.collection.video.v1" - ) - with connect() as conn: - with pytest.raises(psycopg.Error) as caught: - with conn.transaction(): - commit( - conn, - namespace, - first, - [record("new")], - expected={"page": 1}, - next_state={"page": 2}, - root_key="absent", - ) - assert caught.value.sqlstate == "22023" - assert fetch_one( - conn.execute("SELECT block,state FROM inkcre.sources WHERE id=%s", (first,)) - ) == (first_anchor, {"page": 1}) - assert ( - fetch_one( - conn.execute( - "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) - ) - )[0] - == 1 - ) - # Seven positional arguments still resolve to the one eight-argument function. - commit(conn, namespace, first, [], expected={"page": 1}, next_state={"page": 2}) - assert fetch_one( - conn.execute("SELECT block,state FROM inkcre.sources WHERE id=%s", (first,)) - ) == (first_anchor, {"page": 2}) - - -def test_incompatible_source_anchor_never_rewrites_business_block(scope): - namespace, (source_id, _) = scope - with connect() as conn: - result = commit(conn, namespace, source_id, [record(title="business")]) - video = result["blocks"]["video:1"] - conn.execute("UPDATE inkcre.sources SET block=%s WHERE id=%s", (video, source_id)) - with connect() as conn: - with pytest.raises(psycopg.Error) as caught: - with conn.transaction(): - commit( - conn, - namespace, - source_id, - [record(title="must roll back")], - root_key="video:1", - next_state={"page": 1}, - ) - assert caught.value.sqlstate == "22023" - stored = fetch_one( - conn.execute("SELECT resolver,content FROM inkcre.blocks WHERE id=%s", (video,)) - ) - assert stored[0] == "tests.collection.video.v1" - assert json.loads(stored[1])["title"] == "business" - assert ( - fetch_one(conn.execute("SELECT state FROM inkcre.sources WHERE id=%s", (source_id,)))[ - 0 - ] - == {} - ) - - -@pytest.mark.parametrize("failure", ["state", "config", "resolver", "endpoint"]) -def test_conflict_or_invalid_graph_rolls_back_entire_batch(scope, failure): - namespace, (source_id, _) = scope - with connect() as conn: - initial = commit( - conn, namespace, source_id, [record(title="before")], next_state={"page": 1} - ) - with connect() as conn: - with pytest.raises(psycopg.Error) as caught: - with conn.transaction(): - changed = record(title="wrong") - if failure == "resolver": - changed["resolver"] = "tests.collection.incompatible.v1" - commit( - conn, - namespace, - source_id, - [record("video:new"), changed], - expected={} if failure == "state" else {"page": 1}, - config={"changed": True} if failure == "config" else {}, - next_state={"page": 2}, - relations=[{"from_key": "video:1", "to_key": "absent", "content": "contains"}] - if failure == "endpoint" - else [], - ) - assert caught.value.sqlstate == ("PT409" if failure in {"state", "config"} else "22023") - assert ( - fetch_one( - conn.execute( - "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) - ) - )[0] - == 1 - ) - assert fetch_one( - conn.execute("SELECT state FROM inkcre.sources WHERE id=%s", (source_id,)) - )[0] == {"page": 1} - assert ( - json.loads( - fetch_one( - conn.execute( - "SELECT content FROM inkcre.blocks WHERE id=%s", (initial["blocks"]["video:1"],) - ) - )[0] - )["title"] - == "before" - ) - - -@pytest.mark.parametrize( - "records", - [ - [record(), record()], - [record("")], - [{"key": "bad", "resolver": "x", "content": []}], - [record()] * 501, - ], -) -def test_rejects_invalid_or_oversized_records(scope, records): - namespace, (source_id, _) = scope - with connect() as conn: - with pytest.raises(psycopg.Error) as caught: - with conn.transaction(): - commit(conn, namespace, source_id, records) - assert caught.value.sqlstate == "22023" - assert ( - fetch_one( - conn.execute( - "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) - ) - )[0] - == 0 - ) - - -def test_concurrent_sources_reuse_native_identity(scope): - namespace, source_ids = scope - barrier = Barrier(2) - - def execute(source_id): - with connect() as conn: - barrier.wait(timeout=10) - return commit( - conn, namespace, source_id, [record(title="same")], next_state={"done": True} - )["blocks"]["video:1"] - - with ThreadPoolExecutor(max_workers=2) as pool: - ids = list(pool.map(execute, source_ids)) - assert ids[0] == ids[1] - with connect() as conn: - assert ( - fetch_one( - conn.execute( - "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) - ) - )[0] - == 1 - ) - - -def test_rpc_is_invoker_and_does_not_grant_public_access(scope): - with connect() as conn: - row = fetch_one( - conn.execute( - "SELECT p.prosecdef, p.proconfig, " - "EXISTS (SELECT 1 FROM aclexplode(p.proacl) acl " - "WHERE acl.grantee=0 AND acl.privilege_type='EXECUTE') " - "FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace " - "WHERE n.nspname='inkcre' AND p.proname='commit_source_batch'" - ) - ) - assert row == (False, ["search_path=pg_catalog"], False) - - -def test_authenticated_executes_with_ordinary_permissions_and_anonymous_cannot(scope): - namespace, source_ids = scope - with connect() as conn: - if ( - fetch_one( - conn.execute( - "SELECT count(*) FROM pg_roles WHERE rolname IN ('authenticated', 'anonymous')" - ) - )[0] - != 2 - ): - pytest.skip("requires database lifecycle principals in the disposable cluster") - # These grants are local to the disposable test database, not cluster role changes. - conn.execute("GRANT USAGE ON SCHEMA inkcre TO authenticated") - conn.execute( - "GRANT SELECT,INSERT,UPDATE ON inkcre.sources, inkcre.blocks, " - "inkcre.relations, inkcre.collection_records TO authenticated" - ) - conn.execute("GRANT SELECT ON inkcre.storages, inkcre.storage_types TO authenticated") - conn.execute("GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA inkcre TO authenticated") - conn.execute( - "GRANT EXECUTE ON FUNCTION inkcre.commit_source_batch" - "(integer,text,jsonb,jsonb,jsonb,jsonb,jsonb,text) TO authenticated" - ) - conn.execute("SET LOCAL ROLE authenticated") - result = commit(conn, namespace, source_ids[0], [record()], next_state={"ok": True}) - assert result["blocks"]["video:1"] > 0 - with connect() as conn: - with pytest.raises(psycopg.errors.InsufficientPrivilege): - with conn.transaction(): - conn.execute("SET LOCAL ROLE anonymous") - commit(conn, namespace, source_ids[1], [record("denied")]) - assert ( - fetch_one( - conn.execute( - "SELECT count(*) FROM inkcre.collection_records WHERE namespace=%s", (namespace,) - ) - )[0] - == 1 - ) - - -def test_protocol_projection_declares_the_same_rpc_and_native_mapping(): - from app.database_contract.protocol import ( - protocol_database_function_signatures, - protocol_document, - ) - - relations = protocol_document()["relations"] - assert isinstance(relations, dict) - assert "collection_records" in relations - names, types, returns, returns_set, volatility = protocol_database_function_signatures()[ - "commit_source_batch" - ] - assert names == ( - "source_id", - "namespace", - "expected_config", - "expected_state", - "next_state", - "records", - "relations", - "root_key", - ) - assert types == ("integer", "text", "jsonb", "jsonb", "jsonb", "jsonb", "jsonb", "text") - assert (returns, returns_set, volatility) == ("jsonb", False, "v") diff --git a/tests/test_dev_database_provider.py b/tests/test_dev_database_provider.py index e2881431..b044f84a 100644 --- a/tests/test_dev_database_provider.py +++ b/tests/test_dev_database_provider.py @@ -1,50 +1,10 @@ -import json from io import BytesIO +import subprocess import tarfile from scripts import dev_database_provider as provider -def test_schema5_machine_override_selects_the_declared_ssh_provider(tmp_path, monkeypatch): - base = tmp_path / "svc.json" - local = tmp_path / "svc.local.json" - base.write_text(json.dumps({"schema_version": 5, "dev": {"resources": {}}})) - local.write_text( - json.dumps( - { - "schema_version": 5, - "dev": { - "resources": { - "database": { - "actions": { - "ensure": { - "env": { - "INKCRE_DATABASE_PROVIDER": "ssh", - "INKCRE_DATABASE_SSH_TARGET": "test-host", - "INKCRE_DATABASE_SSH_DOCKER_BIN": "/usr/bin/docker", - } - } - } - } - } - }, - } - ) - ) - monkeypatch.setattr(provider, "BASE_CONFIG", base) - monkeypatch.setattr(provider, "LOCAL_CONFIG", local) - for name in ( - "INKCRE_DATABASE_PROVIDER", - "INKCRE_DATABASE_SSH_TARGET", - "INKCRE_DATABASE_SSH_DOCKER_BIN", - ): - monkeypatch.delenv(name, raising=False) - actual = provider.resolve_database_provider() - assert actual.kind == "ssh" - assert actual.target == "test-host" - assert actual.docker_bin == "/usr/bin/docker" - - def test_remote_build_context_carries_the_sdk_and_required_license(tmp_path): environment = tmp_path / "compose.env" environment.write_text("INKCRE_COMPOSE_PROJECT_NAME='test-context'\n") @@ -53,3 +13,23 @@ def test_remote_build_context_carries_the_sdk_and_required_license(tmp_path): names = set(archive.getnames()) assert "context/inkcre_core/__init__.py" in names assert "context/LICENSE" in names + + +def test_compose_failure_keeps_both_streams_and_redacts_credentials(tmp_path): + environment = tmp_path / "compose.env" + environment.write_text("POSTGRES_PASSWORD='test-only-secret'\n") + error = subprocess.CalledProcessError( + 1, + ["docker"], + output=b"Actual build failure: test-only-secret", + stderr=b"Compose warning", + ) + diagnostic = str( + provider._compose_failure( + error, environment, provider.DatabaseProvider(kind="ssh", target="test-host") + ) + ) + assert "Compose warning" in diagnostic + assert "Actual build failure" in diagnostic + assert "test-only-secret" not in diagnostic + assert "[REDACTED]" in diagnostic From b4d1143e809a3001d9d7d5cf1ebc54f918006904 Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Mon, 9 Nov 2026 21:54:45 +0800 Subject: [PATCH 6/6] docs(shared): pin merged Bilibili contract - Pin the shared docs to the squash commit from InKCre/docs#59. --- docs/_shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_shared b/docs/_shared index b50921b1..ad76242b 160000 --- a/docs/_shared +++ b/docs/_shared @@ -1 +1 @@ -Subproject commit b50921b19cb6cf81ad4c09786423195974d901b2 +Subproject commit ad76242bd3a10536001aa751c1152bec7fb42990