From 149230738b7615ab9b01be27c2a9b20d60972910 Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 00:02:54 +0800 Subject: [PATCH 01/12] =?UTF-8?q?feat(collection):=20=E6=96=B0=E5=A2=9E=20?= =?UTF-8?q?B=20=E7=AB=99=E6=94=B6=E8=97=8F=E5=A4=B9=E4=B8=8E=E6=B5=8F?= =?UTF-8?q?=E8=A7=88=E5=99=A8=E5=8D=8F=E5=90=8C=E9=87=87=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 为 Web Source 提供精确浏览器绑定、Puppeteer 页面能力和本机消息桥 - 新增 B 站收藏夹采集、批次持久化消费者与独立读取视图 - 补充生命周期、取消、合同生成和隔离候选验证 --- .changeset/bilibili-favorites.md | 5 + .changeset/web-source-browser-automation.md | 8 + .github/preview/extensions.json | 5 + .github/workflows/ci.yml | 20 +- .gitignore | 2 + THIRD_PARTY_NOTICES.txt | 287 +++++++++ apps/web/src/core.ts | 3 + apps/webext/README.md | 15 +- .../components/BrowserAutomationSettings.vue | 88 +++ apps/webext/entrypoints/background.ts | 2 + .../entrypoints/browser-bridge.content.ts | 84 +++ apps/webext/entrypoints/options/Options.vue | 2 + .../logic/browser-automation/runtime.ts | 430 +++++++++++++ apps/webext/package.json | 1 + apps/webext/wxt.config.ts | 7 +- docs/30-unit-tdd/README.md | 2 + docs/30-unit-tdd/bilibili-extension.md | 33 + docs/30-unit-tdd/browser-automation.md | 63 ++ docs/30-unit-tdd/info-base.md | 6 + docs/40-deployment/development-runtime.md | 21 +- .../native-extension-delivery.md | 13 +- extensions/bilibili/CHANGELOG.md | 7 + extensions/bilibili/LICENSE | 21 + extensions/bilibili/README.md | 42 ++ extensions/bilibili/index.html | 12 + extensions/bilibili/package.json | 48 ++ extensions/bilibili/src/BilibiliContent.vue | 175 +++++ extensions/bilibili/src/BilibiliPreview.vue | 50 ++ extensions/bilibili/src/BilibiliSetup.vue | 600 ++++++++++++++++++ extensions/bilibili/src/DevApp.vue | 8 + extensions/bilibili/src/collect.ts | 63 ++ extensions/bilibili/src/graph.ts | 50 ++ extensions/bilibili/src/index.ts | 31 + extensions/bilibili/src/main.ts | 8 + extensions/bilibili/src/normalize.ts | 200 ++++++ extensions/bilibili/src/progress.ts | 163 +++++ extensions/bilibili/src/repository.ts | 37 ++ extensions/bilibili/src/resolver.ts | 149 +++++ extensions/bilibili/src/schema.ts | 121 ++++ extensions/bilibili/src/site.ts | 77 +++ extensions/bilibili/src/source.ts | 34 + extensions/bilibili/tests/favorites.test.ts | 366 +++++++++++ extensions/bilibili/tsconfig.json | 19 + extensions/bilibili/vite.config.ts | 73 +++ packages/core/package.json | 5 + .../core/src/browser-automation/access.ts | 296 +++++++++ packages/core/src/browser-automation/index.ts | 3 + .../core/src/browser-automation/protocol.ts | 61 ++ .../core/src/database/database.generated.ts | 39 ++ .../database/runtime-contract.generated.json | 2 +- packages/core/src/index.ts | 3 + .../core/src/info-base/resolvers/contracts.ts | 1 + .../core/src/info-base/resolvers/index.ts | 2 + packages/core/src/job/manager.ts | 10 +- packages/core/src/source/index.ts | 1 + packages/core/src/source/manager.ts | 20 +- packages/core/src/source/resolver.ts | 38 ++ packages/core/tsdown.config.ts | 10 +- patches/puppeteer-core@25.16.0.patch | 51 ++ pnpm-lock.yaml | 170 ++++- pnpm-workspace.yaml | 2 + scripts/check-local-runtime-contract.mjs | 18 +- scripts/check-package-contract.mjs | 1 + scripts/check-workspace-contract.mjs | 6 + scripts/database-contract-source.mjs | 16 + scripts/database-provider.mjs | 6 +- scripts/doctor.mjs | 2 +- scripts/prepare-native-checkout.mjs | 190 ++++++ scripts/sync-database-contract.mjs | 65 +- scripts/test-webext-e2e.mjs | 7 + .../browser-automation-transport.test.mjs | 78 +++ scripts/tests/browser-automation/index.html | 11 + scripts/tests/browser-automation/main.ts | 250 ++++++++ .../tests/database-contract-source.test.mjs | 38 ++ scripts/tests/job-cancellation.test.mjs | 86 +++ .../tests/prepare-native-checkout.test.mjs | 187 ++++++ scripts/tests/source-anchor.test.mjs | 36 ++ scripts/verify-browser-automation.mjs | 213 +++++++ scripts/webext-release.mjs | 3 +- svc.json | 190 ++++-- tests/e2e/browser-extension.spec.ts | 2 +- 81 files changed, 5457 insertions(+), 113 deletions(-) create mode 100644 .changeset/bilibili-favorites.md create mode 100644 .changeset/web-source-browser-automation.md create mode 100644 apps/webext/components/BrowserAutomationSettings.vue create mode 100644 apps/webext/entrypoints/browser-bridge.content.ts create mode 100644 apps/webext/logic/browser-automation/runtime.ts create mode 100644 docs/30-unit-tdd/bilibili-extension.md create mode 100644 docs/30-unit-tdd/browser-automation.md create mode 100644 extensions/bilibili/CHANGELOG.md create mode 100644 extensions/bilibili/LICENSE create mode 100644 extensions/bilibili/README.md create mode 100644 extensions/bilibili/index.html create mode 100644 extensions/bilibili/package.json create mode 100644 extensions/bilibili/src/BilibiliContent.vue create mode 100644 extensions/bilibili/src/BilibiliPreview.vue create mode 100644 extensions/bilibili/src/BilibiliSetup.vue create mode 100644 extensions/bilibili/src/DevApp.vue create mode 100644 extensions/bilibili/src/collect.ts create mode 100644 extensions/bilibili/src/graph.ts create mode 100644 extensions/bilibili/src/index.ts create mode 100644 extensions/bilibili/src/main.ts create mode 100644 extensions/bilibili/src/normalize.ts create mode 100644 extensions/bilibili/src/progress.ts create mode 100644 extensions/bilibili/src/repository.ts create mode 100644 extensions/bilibili/src/resolver.ts create mode 100644 extensions/bilibili/src/schema.ts create mode 100644 extensions/bilibili/src/site.ts create mode 100644 extensions/bilibili/src/source.ts create mode 100644 extensions/bilibili/tests/favorites.test.ts create mode 100644 extensions/bilibili/tsconfig.json create mode 100644 extensions/bilibili/vite.config.ts create mode 100644 packages/core/src/browser-automation/access.ts create mode 100644 packages/core/src/browser-automation/index.ts create mode 100644 packages/core/src/browser-automation/protocol.ts create mode 100644 packages/core/src/source/resolver.ts create mode 100644 patches/puppeteer-core@25.16.0.patch create mode 100644 scripts/database-contract-source.mjs create mode 100644 scripts/prepare-native-checkout.mjs create mode 100644 scripts/tests/browser-automation-transport.test.mjs create mode 100644 scripts/tests/browser-automation/index.html create mode 100644 scripts/tests/browser-automation/main.ts create mode 100644 scripts/tests/database-contract-source.test.mjs create mode 100644 scripts/tests/job-cancellation.test.mjs create mode 100644 scripts/tests/prepare-native-checkout.test.mjs create mode 100644 scripts/tests/source-anchor.test.mjs create mode 100644 scripts/verify-browser-automation.mjs diff --git a/.changeset/bilibili-favorites.md b/.changeset/bilibili-favorites.md new file mode 100644 index 00000000..61103a4a --- /dev/null +++ b/.changeset/bilibili-favorites.md @@ -0,0 +1,5 @@ +--- +'@inkcre/ext-bilibili': minor +--- + +新增 B 站收藏夹插件，通过所选浏览器采集自己的公开和私密收藏夹，保存原生身份、文字资料和成员关系，并提供独立读取视图。支持有界分批与进度复核，不保存视频二进制。 diff --git a/.changeset/web-source-browser-automation.md b/.changeset/web-source-browser-automation.md new file mode 100644 index 00000000..3d42d459 --- /dev/null +++ b/.changeset/web-source-browser-automation.md @@ -0,0 +1,8 @@ +--- +'@inkcre/core': patch +'@inkcre/client-webext': patch +--- + +为 Web Source 提供基于 Puppeteer 的本机浏览器能力，支持明确的 Web origin 授权、精确浏览器绑定、登录态页面交互与网络响应读取。 + +Chromium 扩展新增 debugger 权限，最低 Chrome 125。新建页与借用页分别清理，取消和断连不会自动重放操作。普通 Source Job 通过业务资格钩子选择执行 Peer。 diff --git a/.github/preview/extensions.json b/.github/preview/extensions.json index ccc70f2a..903ce8d6 100644 --- a/.github/preview/extensions.json +++ b/.github/preview/extensions.json @@ -1,6 +1,11 @@ { "schema_version": 1, "distributions": [ + { + "kind": "module_federation", + "producer": "../../extensions/bilibili/package.json", + "artifact": "../../extensions/bilibili/dist/client-web" + }, { "kind": "module_federation", "producer": "../../extensions/caldav/package.json", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6586dd07..d5b4e081 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,16 +110,32 @@ jobs: - name: Install the frozen workspace run: pnpm install --frozen-lockfile - - name: 校验 TS7、Vue 类型、静态规则及产物合同 - run: pnpm check + - name: 校验源代码格式、静态规则和类型 + run: pnpm format:check && pnpm lint && pnpm type-check + + - name: 以正式版本命令准备隔离验证坐标 + run: node scripts/prepare-native-checkout.mjs --output "$RUNNER_TEMP/native-checkout" + + - name: 构建并校验相互兼容的 Host 与 MF 产物 + working-directory: ${{ runner.temp }}/native-checkout/workspace + run: pnpm build + + - name: 验证 B 站收藏采集规则 + working-directory: ${{ runner.temp }}/native-checkout/workspace + run: pnpm --filter @inkcre/ext-bilibili test - name: Verify independent MF publication + working-directory: ${{ runner.temp }}/native-checkout/workspace run: >- node --test scripts/test-extension-publication.mjs scripts/test-core-sdk-publication.mjs scripts/tests/independent-extension-release.test.mjs scripts/tests/release-checkout.test.mjs scripts/tests/core-sdk-compatibility.test.mjs scripts/tests/pages-preview.test.mjs + scripts/tests/browser-automation-transport.test.mjs + scripts/tests/job-cancellation.test.mjs + scripts/tests/source-anchor.test.mjs + scripts/tests/prepare-native-checkout.test.mjs peer-database: # Existing branch protection binds this exact check name. diff --git a/.gitignore b/.gitignore index 39ff2a0b..755fefcb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ node_modules package-lock.json # Build output +.output/ +.wxt/ dist dist-ssr coverage diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index abef4e3c..d80f9469 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -832,3 +832,290 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice This Source Code Form is "Incompatible With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. + + +====================================================================== +puppeteer-core 25.16.0 +https://github.com/puppeteer/puppeteer + +Copyright 2017 Google Inc. +SPDX-License-Identifier: Apache-2.0 +The complete Apache-2.0 text is retained below with Puppeteer's bundled RxJS dependency. + + +====================================================================== +mitt (bundled by puppeteer-core 25.16.0) +MIT License + +Copyright (c) 2021 Jason Miller + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +====================================================================== +parsel-js (bundled by puppeteer-core 25.16.0) +MIT License + +Copyright (c) 2020 Lea Verou + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +====================================================================== +urlpattern-polyfill (bundled by puppeteer-core 25.16.0) +Copyright 2020 Intel Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +====================================================================== +rxjs (bundled by puppeteer-core 25.16.0) +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2015-2018 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/apps/web/src/core.ts b/apps/web/src/core.ts index 95fd7c8b..db3cd373 100644 --- a/apps/web/src/core.ts +++ b/apps/web/src/core.ts @@ -17,6 +17,7 @@ import { localStorageAdapter, registerCoreResolvers, TextResolver, + SourceResolver, MarkdownResolver, ResolverCache, AudioResolver, @@ -122,6 +123,7 @@ export function setupResolvers(): void { MarkdownResolver.previewRenderer = ContentPreview MarkdownResolver.solvedContentRenderer = ContentHtml TextResolver.previewRenderer = ContentPreview + SourceResolver.previewRenderer = ContentPreview AudioResolver.previewRenderer = ContentPreview EpubResolver.previewRenderer = ContentPreview FileResolver.previewRenderer = ContentPreview @@ -131,6 +133,7 @@ export function setupResolvers(): void { HtmlResolver.previewRenderer = ContentPreview ZipResolver.previewRenderer = ContentPreview TextResolver.solvedContentRenderer = ContentText + SourceResolver.solvedContentRenderer = ContentText AudioResolver.solvedContentRenderer = ContentAudio EpubResolver.solvedContentRenderer = ContentFile FileResolver.solvedContentRenderer = ContentFile diff --git a/apps/webext/README.md b/apps/webext/README.md index 604046ea..d9d44226 100644 --- a/apps/webext/README.md +++ b/apps/webext/README.md @@ -16,7 +16,7 @@ HTML 和文字是普通 core 语义内容块，PDF 使用当前 Peer 支持的 本机待发送与已保存到信息库是不同状态。待办绑定入队时的信息库与本 Peer 身份，后台自行推进，不靠 popup 或设置页轮询触发；关闭界面或重启后仍可继续。暂停和排除停止未派发的自动工作，手动保存仍可使用；排除编辑器回填现有规则，刷新状态不覆盖未保存的编辑。恢复尽力复用可见图结果，但不承诺 exactly-once、全图事务或完整历史。响应丢失、并发或中途退出可能留下重复或孤立项，失败不会被显示为完整保存。 -书签和历史先预览范围，确认后保存来源事实并尝试取得当前正文。记录时间与当前取得时间分别保留。登录、失效或不可读项保留来源和原因，支持停止、继续和失败重试；原书签删除不自动删除信息库资料。第三方收藏服务连接器仍待选定真实资料源。 +书签和历史先预览范围，确认后保存来源事实并尝试取得当前正文。记录时间与当前取得时间分别保留。登录、失效或不可读项保留来源和原因，支持停止、继续和失败重试；原书签删除不自动删除信息库资料。第三方收藏服务由独立插件提供，Web-ext 本身不保存站点选择器或收藏模型。 阅读页提供文本、出处、附注、HTML 源码和原始 PDF 下载，不执行来源脚本。不承诺取得尚未加载、虚拟化移除、canvas、封闭组件或不可读跨域 frame 的正文；已发现缺口会标注。 @@ -24,6 +24,17 @@ Explain、Writing Assist、旧选区浮栏、图编辑侧栏和模型设置已 升级时重新加载扩展，并刷新此前已注入旧浮栏的网页，让旧版本留在页面中的内容一并清除。 +## 让 Web Source 使用浏览器 + +Chromium 125 及以上版本可为 InKCre Web 中运行的 Source 提供浏览器能力。安装或升级这版扩展需要原生 debugger 权限；具体 Web 地址仍须由你单独授权。 + +1. 在扩展设置中配置当前信息库连接。 +2. 在“浏览器操作”输入可信任的 InKCre Web 地址，点击“授权此站点”并完成原生站点权限操作。 +3. 回到 Web 的业务插件设置，查找并明确绑定此浏览器，再选择要采集的范围。B 站等网站的登录直接在该浏览器中完成，无需向 Source 复制 cookie。 +4. 从 Web 运行普通采集任务。扩展负责真实标签页操作，业务插件负责解释和保存信息。 + +可在扩展设置撤销某个 Web 地址的授权，这会停止该授权下的浏览器会话。开始操作时会切换该浏览器窗口的活动标签页；采集新建的页面结束后关闭，借用的已有页面保留。Firefox 版本仍提供既有材料采集，暂不提供这条自动化路径。当前不承诺关闭 Web 或浏览器后继续运行采集任务。 + ## 开发与验证 使用根仓库声明的 Node.js/pnpm 版本，运行 `pnpm install --frozen-lockfile`。本目录的 `pnpm dev`、`pnpm build` 面向 Chromium，`pnpm dev:firefox`、`pnpm build:firefox` 面向 Firefox；`pnpm type-check` 检查扩展。 @@ -36,7 +47,7 @@ Explain、Writing Assist、旧选区浮栏、图编辑侧栏和模型设置已 WebExt 固定使用正式发布的 `@inkcre/ui-web` 2.6.0，通过内置密码显隐能力、公开 `INK_I18N_KEY` 和中文词典提供按钮名称，不另写密码组件。2.5.0 和 2.6.0 的普通可编辑输入框都会额外渲染只读值和重复 ID，因此仓库分别维护版本限定的 pnpm patch，各修正一个渲染条件；升级到上游修复版后移除对应补丁。这次仅 WebExt 升级，web、twitter 和 ext-dev-utils 继续使用原 2.5.0 及其补丁，不改变组件 API、样式或仍使用 2.3.0 的扩展。 -根目录 `pnpm test:e2e:webext` 真正加载扩展。指定 `INKCRE_CHROMIUM_BINARY=/Applications/Helium.app/Contents/MacOS/Helium` 使用 Helium 的隔离测试 profile，测试结束清理，不连接日常会话。真实数据旅程还需要可销毁数据库/PostgREST 的 `INKCRE_E2E_WEBEXT_PGREST` 与 `INKCRE_E2E_WEBEXT_JWT_SECRET`。测试副本只对合成 localhost 资料预授予权限；原生权限 UI 另行验证，生产 manifest 保持 optional 权限。 +根目录 `pnpm test:e2e:webext` 真正加载扩展。指定 `INKCRE_CHROMIUM_BINARY=/Applications/Helium.app/Contents/MacOS/Helium` 使用 Helium 的隔离测试 profile，测试结束清理，不连接日常会话。真实数据旅程还需要可销毁数据库/PostgREST 的 `INKCRE_E2E_WEBEXT_PGREST` 与 `INKCRE_E2E_WEBEXT_JWT_SECRET`。测试副本只对合成 localhost 资料预授予权限；原生权限 UI 另行验证，合成 host 的预授权不进入生产 manifest；Chromium 的 debugger 为必选权限，站点权限仍按需申请。 实现合同见[WebExt 运行时与材料](../../docs/30-unit-tdd/webext-collection.md)。安装、手动更新、旧身份迁移和 Changesets/GitHub Release 流程见[WebExt 版本与分发](../../docs/40-deployment/webext-delivery.md)。本地构建不等于发布；Firefox ZIP 尚未签名，不能作为正式安装包。 diff --git a/apps/webext/components/BrowserAutomationSettings.vue b/apps/webext/components/BrowserAutomationSettings.vue new file mode 100644 index 00000000..5df59e19 --- /dev/null +++ b/apps/webext/components/BrowserAutomationSettings.vue @@ -0,0 +1,88 @@ + + + diff --git a/apps/webext/entrypoints/background.ts b/apps/webext/entrypoints/background.ts index d860b024..1c4c3542 100644 --- a/apps/webext/entrypoints/background.ts +++ b/apps/webext/entrypoints/background.ts @@ -1,4 +1,5 @@ import { browser } from 'wxt/browser' +import { initializeBrowserAutomation } from '~/logic/browser-automation/runtime' import { handleCollectionMessage, handlesCollectionMessage, @@ -9,6 +10,7 @@ import { } from '~/logic/collection/runtime' export default defineBackground(() => { + initializeBrowserAutomation() const initialized = initializeCollection() initialized .then(() => Promise.all([wakeWork(), refreshPeers()])) diff --git a/apps/webext/entrypoints/browser-bridge.content.ts b/apps/webext/entrypoints/browser-bridge.content.ts new file mode 100644 index 00000000..035821a7 --- /dev/null +++ b/apps/webext/entrypoints/browser-bridge.content.ts @@ -0,0 +1,84 @@ +import { browser, type Browser } from 'wxt/browser' +import { + BROWSER_AUTOMATION_PORT, + BROWSER_AUTOMATION_PROTOCOL, + BrowserBridgeRequestSchema, + BrowserBridgeResponseSchema, + BrowserBridgeEventSchema, + type BrowserBridgeRequest, + type BrowserBridgeEvent, +} from '@inkcre/core/browser-automation/protocol' + +export default defineContentScript({ + matches: [], + registration: 'runtime', + main(ctx) { + // Granting a second information-base binding may inject the bridge again in the same page. + const marker = '__inkcreBrowserBridgeV1' + const scope = globalThis as typeof globalThis & { [marker]?: boolean } + if (scope[marker]) return + scope[marker] = true + let port: Browser.runtime.Port | undefined + const pending = new Map() + const sessions = new Map>() + const disconnected = () => { + for (const request of pending.values()) { + if (!request.provider || !request.instance) continue + window.postMessage( + { + ...request, + kind: 'response', + ok: false, + error: '浏览器桥已断开；已发送操作的结果可能未知。', + }, + location.origin + ) + } + for (const session of sessions.values()) + window.postMessage({ ...session, event: 'closed' }, location.origin) + pending.clear() + sessions.clear() + port = undefined + } + ctx.addEventListener(window, 'message', (event: MessageEvent) => { + if (event.source !== window || event.origin !== location.origin) return + const request = BrowserBridgeRequestSchema.safeParse(event.data) + if (!request.success) return + if (!port) { + port = browser.runtime.connect({ name: BROWSER_AUTOMATION_PORT }) + port.onMessage.addListener((reply: unknown) => { + const response = BrowserBridgeResponseSchema.safeParse(reply) + if (response.success) { + const outgoing = pending.get(response.data.request) + const opened = response.data.value as { session?: string } | undefined + if (outgoing?.operation === 'open' && response.data.ok && opened?.session) + sessions.set(opened.session, { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'event', + client: response.data.client, + provider: response.data.provider, + instance: response.data.instance, + session: opened.session, + }) + pending.delete(response.data.request) + } + const event = BrowserBridgeEventSchema.safeParse(reply) + if (event.success && event.data.event === 'closed') sessions.delete(event.data.session) + window.postMessage(reply, location.origin) + }) + port.onDisconnect.addListener(disconnected) + } + try { + pending.set(request.data.request, request.data) + port.postMessage(request.data) + } catch { + disconnected() + } + }) + ctx.addEventListener(window, 'pagehide', () => port?.disconnect()) + ctx.onInvalidated(() => { + port?.disconnect() + delete scope[marker] + }) + }, +}) diff --git a/apps/webext/entrypoints/options/Options.vue b/apps/webext/entrypoints/options/Options.vue index 20b48baf..9f4ec580 100644 --- a/apps/webext/entrypoints/options/Options.vue +++ b/apps/webext/entrypoints/options/Options.vue @@ -4,6 +4,7 @@ import { browser } from 'wxt/browser' import { InkButton, InkDropdown, InkForm, InkInput, InkSwitch, InkTextarea } from '@inkcre/ui-web' import type { MaterialRead } from '@inkcre/web-collection' import MaterialReader from '~/components/collection/MaterialReader.vue' +import BrowserAutomationSettings from '~/components/BrowserAutomationSettings.vue' import logo from '@inkcre/ui-web/logo/svg/original-transparent/inkcre-original-transparent-32.svg?url' import reverseLogo from '@inkcre/ui-web/logo/svg/reverse-transparent/inkcre-reverse-transparent-32.svg?url' import { collectionMessage } from '~/logic/collection/messages' @@ -202,6 +203,7 @@ onMounted(() => { +

信息库

crypto.randomUUID()), + grants: z.array(z.object({ origin: z.url(), provider: z.uuid() })).default([]), +}) +type State = z.infer +type Port = Browser.runtime.Port +type Session = { + id: string + provider: string + client: string + port: Port + tabId: number + created: boolean + transport: ExtensionTransport + detached: boolean + closing?: Promise +} +const epoch = crypto.randomUUID() +const sessions = new Map() +const occupiedTabs = new Set() +const ports = new Map() +const tabReferences = new Map() + +export async function automationState(): Promise { + const stored = await browser.storage.local.get(STORAGE_KEY) + const state = StateSchema.parse(stored[STORAGE_KEY] ?? {}) + if (!stored[STORAGE_KEY]) await browser.storage.local.set({ [STORAGE_KEY]: state }) + return state +} + +function originOf(value: string): string { + const url = new URL(value) + if (!['http:', 'https:'].includes(url.protocol)) throw new Error('只支持 HTTP(S) Web 地址。') + if (url.username || url.password) throw new Error('Web 地址不能包含凭据。') + return url.origin +} + +export async function grantAutomation(origin: string, provider: string): Promise { + if (!import.meta.env.CHROME) throw new Error('此浏览器不支持调试自动化；请使用 Chromium。') + const exactOrigin = originOf(origin) + if (!(await connections()).some((target) => target.peerId === provider)) + throw new Error('请先为此浏览器配置所选信息库连接。') + if (!(await browser.permissions.contains({ origins: [siteMatchPattern(exactOrigin)] }))) + throw new Error('请先授权此 Web 站点。') + const state = await automationState() + state.grants = [ + ...state.grants.filter((grant) => grant.origin !== exactOrigin || grant.provider !== provider), + { origin: exactOrigin, provider }, + ] + await browser.storage.local.set({ [STORAGE_KEY]: state }) + await syncAutomationScripts() + for (const tab of await browser.tabs.query({})) { + if (tab.id === undefined || !tab.url || new URL(tab.url).origin !== exactOrigin) continue + await browser.scripting + .executeScript({ target: { tabId: tab.id }, files: ['/content-scripts/browser-bridge.js'] }) + .catch(() => undefined) + } +} + +export async function revokeAutomation(origin: string, provider: string): Promise { + const state = await automationState() + state.grants = state.grants.filter( + (grant) => grant.origin !== origin || grant.provider !== provider + ) + await browser.storage.local.set({ [STORAGE_KEY]: state }) + await Promise.all( + [...sessions.values()] + .filter((session) => ports.get(session.port) === origin && session.provider === provider) + .map((session) => release(session)) + ) + if (!state.grants.some((grant) => grant.origin === origin)) + for (const [port, portOrigin] of ports) if (portOrigin === origin) port.disconnect() + await syncAutomationScripts() +} + +export async function syncAutomationScripts(): Promise { + const registered = (await browser.scripting.getRegisteredContentScripts()) + .filter((script) => script.id.startsWith('inkcre-browser-')) + .map((script) => script.id) + if (registered.length) await browser.scripting.unregisterContentScripts({ ids: registered }) + if (!import.meta.env.CHROME) return + const state = await automationState() + const matches: string[] = [] + for (const grant of state.grants) { + const pattern = siteMatchPattern(grant.origin) + if (!matches.includes(pattern) && (await browser.permissions.contains({ origins: [pattern] }))) + matches.push(pattern) + } + if (!matches.length) return + await browser.scripting.registerContentScripts([ + { + id: 'inkcre-browser-bridge-v1', + matches, + js: ['content-scripts/browser-bridge.js'], + runAt: 'document_start', + allFrames: false, + persistAcrossSessions: true, + }, + ]) +} + +function post(port: Port, value: unknown): void { + try { + port.postMessage(value) + } catch { + // A disappearing Web page is handled by the Port's disconnect cleanup. + } +} + +function release(session: Session): Promise { + if (session.closing) return session.closing + if (!sessions.has(session.id)) return Promise.resolve() + session.closing = (async () => { + try { + await session.transport.close() + } catch (error) { + if (!session.detached) throw error + } finally { + if (session.created) await browser.tabs.remove(session.tabId).catch(() => undefined) + sessions.delete(session.id) + occupiedTabs.delete(session.tabId) + const state = await automationState() + post(session.port, { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'event', + client: session.client, + provider: session.provider, + instance: state.instance, + session: session.id, + event: 'closed', + }) + } + })() + return session.closing +} + +async function providers(origin: string): Promise { + const state = await automationState() + const targets = await connections() + const allowed = state.grants.filter((grant) => grant.origin === origin) + if (!(await browser.permissions.contains({ origins: [siteMatchPattern(origin)] }))) return [] + return targets + .filter((target) => allowed.some((grant) => grant.provider === target.peerId)) + .map((target) => ({ + provider: target.peerId, + instance: state.instance, + name: target.name, + extensionVersion: browser.runtime.getManifest().version, + protocol: 1, + })) +} + +function tabRef(tab: Browser.tabs.Tab, binding: BrowserProviderBinding): BrowserTabRef { + if (tab.id === undefined) throw new Error('浏览器未返回标签页身份。') + return { + provider: binding.provider, + instance: binding.instance, + epoch, + reference: crypto.randomUUID(), + tabId: tab.id, + title: tab.title, + url: tab.url, + } +} + +async function dispatch( + port: Port, + request: BrowserBridgeRequest, + binding: BrowserProviderBinding +): Promise { + if (request.operation === 'tabs') { + for (const [reference, issued] of tabReferences) + if (issued.port === port && issued.client === request.client) tabReferences.delete(reference) + const tabs = (await browser.tabs.query({})) + .filter((tab) => tab.id !== port.sender?.tab?.id && tab.url && /^https?:\/\//.test(tab.url)) + .map((tab) => tabRef(tab, binding)) + for (const tab of tabs) tabReferences.set(tab.reference, { tab, port, client: request.client }) + return tabs + } + + if (request.operation === 'open') { + const parameters = z + .object({ + session: z.uuid(), + target: z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('new') }), + z.object({ kind: z.literal('existing'), tab: BrowserTabRefSchema }), + ]), + }) + .parse(request.parameters) + if (sessions.has(parameters.session)) throw new Error('浏览器会话身份已被使用。') + const created = parameters.target.kind === 'new' + if (parameters.target.kind === 'existing') { + const ref = parameters.target.tab + if ( + ref.provider !== binding.provider || + ref.instance !== binding.instance || + ref.epoch !== epoch + ) + throw new Error('标签页引用已过期或属于其他浏览器。') + const issued = tabReferences.get(ref.reference) + if ( + !issued || + issued.port !== port || + issued.client !== request.client || + issued.tab.tabId !== ref.tabId + ) + throw new Error('标签页引用已经使用或过期，请刷新标签页列表。') + if (ref.tabId === port.sender?.tab?.id) + throw new Error('不能控制正在运行此 Source 的 Web 标签页。') + if (occupiedTabs.has(ref.tabId)) throw new Error('此标签页正在被另一个自动化会话使用。') + for (const [reference, issued] of tabReferences) + if (issued.tab.tabId === ref.tabId) tabReferences.delete(reference) + occupiedTabs.add(ref.tabId) + } + let tab: Browser.tabs.Tab | undefined + let transport: ExtensionTransport | undefined + try { + tab = created + ? await browser.tabs.create({ url: 'about:blank', active: true }) + : await browser.tabs.update( + parameters.target.kind === 'existing' ? parameters.target.tab.tabId : -1, + { active: true } + ) + if (!tab || tab.id === undefined) throw new Error('无法取得浏览器标签页。') + occupiedTabs.add(tab.id) + transport = await ExtensionTransport.connectTab(tab.id) + const session: Session = { + id: parameters.session, + provider: binding.provider, + client: request.client, + port, + tabId: tab.id, + created, + transport, + detached: false, + } + sessions.set(session.id, session) + transport.onmessage = (data) => + post(port, { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'event', + client: request.client, + provider: binding.provider, + instance: binding.instance, + session: session.id, + event: 'message', + data, + }) + transport.onclose = () => void release(session) + if (!ports.has(port)) { + await release(session) + throw new Error('调用页面已断开，浏览器会话已释放。') + } + return { session: session.id, tab: tabRef(tab, binding) } + } catch (error) { + if (tab?.id !== undefined) { + occupiedTabs.delete(tab.id) + await transport?.close() + if (created) await browser.tabs.remove(tab.id).catch(() => undefined) + } else if (parameters.target.kind === 'existing') + occupiedTabs.delete(parameters.target.tab.tabId) + throw error + } + } + + const sessionId = z.uuid().parse(request.parameters.session) + const session = sessions.get(sessionId) + if (!session) { + if (request.operation === 'close') return null + throw new Error('浏览器会话已结束，旧引用不能再次执行。') + } + if ( + session.port !== port || + session.client !== request.client || + session.provider !== binding.provider + ) + throw new Error('此浏览器会话属于另一个调用者。') + if (request.operation === 'close') { + await release(session) + return null + } + if (request.operation === 'cdp') { + if (session.closing) throw new Error('浏览器会话正在释放。') + session.transport.send(z.string().parse(request.parameters.message)) + return null + } + throw new Error('不支持的浏览器操作。') +} + +async function handle(port: Port, origin: string, request: BrowserBridgeRequest): Promise { + const available = await providers(origin) + if (request.operation === 'discover') { + for (const binding of available) + post(port, { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'response', + client: request.client, + request: request.request, + provider: binding.provider, + instance: binding.instance, + ok: true, + value: binding, + }) + return + } + const binding = available.find( + (item) => item.provider === request.provider && item.instance === request.instance + ) + if (!binding) return + try { + if (!import.meta.env.CHROME) throw new Error('此浏览器不支持调试自动化。') + const value = await dispatch(port, request, binding) + post(port, { + ...request, + kind: 'response', + provider: binding.provider, + instance: binding.instance, + ok: true, + value, + }) + } catch (error) { + post(port, { + ...request, + kind: 'response', + provider: binding.provider, + instance: binding.instance, + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } +} + +export function initializeBrowserAutomation(): void { + void syncAutomationScripts().catch((error) => + console.error('Browser bridge initialization failed:', error) + ) + browser.runtime.onMessage.addListener((raw: unknown, sender) => { + if (!raw || typeof raw !== 'object' || !('type' in raw)) return + if (!String(raw.type).startsWith('browser-automation-')) return + if (sender.id !== browser.runtime.id || !sender.url?.startsWith(browser.runtime.getURL('/'))) + return Promise.resolve({ ok: false, error: '仅扩展设置页面可修改浏览器授权。' }) + const message = raw as { type: string; origin?: string; provider?: string } + const execute = async () => { + if (message.type === 'browser-automation-grant') + await grantAutomation(z.string().parse(message.origin), z.uuid().parse(message.provider)) + else if (message.type === 'browser-automation-revoke') + await revokeAutomation(z.string().parse(message.origin), z.uuid().parse(message.provider)) + else if (message.type !== 'browser-automation-state') throw new Error('未知浏览器设置操作。') + return automationState() + } + return execute().then( + (value) => ({ ok: true, value }), + (error: unknown) => ({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + ) + }) + browser.runtime.onConnect.addListener((port) => { + const sender = port.sender + if ( + port.name !== BROWSER_AUTOMATION_PORT || + sender?.id !== browser.runtime.id || + sender.frameId !== 0 || + sender.tab?.id === undefined || + !sender.url + ) + return + let origin: string + try { + origin = originOf(sender.url) + } catch { + port.disconnect() + return + } + ports.set(port, origin) + // Opening and closing are serialized per page; CDP traffic must keep flowing concurrently. + let lifecycle = Promise.resolve() + port.onMessage.addListener((raw: unknown) => { + const parsed = BrowserBridgeRequestSchema.safeParse(raw) + if (!parsed.success) return + const run = () => handle(port, origin, parsed.data) + if (parsed.data.operation === 'open' || parsed.data.operation === 'close') { + lifecycle = lifecycle.then(run).catch(() => undefined) + } else void run().catch(() => undefined) + }) + port.onDisconnect.addListener(() => { + ports.delete(port) + for (const [reference, issued] of tabReferences) + if (issued.port === port) tabReferences.delete(reference) + for (const session of sessions.values()) if (session.port === port) void release(session) + }) + }) + if (import.meta.env.CHROME) + browser.debugger.onDetach.addListener((source) => { + for (const session of sessions.values()) { + if (session.tabId !== source.tabId) continue + session.detached = true + void release(session).catch(() => undefined) + } + }) + browser.permissions.onRemoved.addListener(() => { + void syncAutomationScripts() + for (const session of sessions.values()) { + const origin = ports.get(session.port) + if (origin) + void providers(origin).then((available) => { + if (!available.some((binding) => binding.provider === session.provider)) + void release(session) + }) + } + }) +} diff --git a/apps/webext/package.json b/apps/webext/package.json index b0cd654d..59405b23 100644 --- a/apps/webext/package.json +++ b/apps/webext/package.json @@ -20,6 +20,7 @@ "@inkcre/ui-web": "2.7.0", "@inkcre/web-collection": "workspace:*", "@mozilla/readability": "^0.6.0", + "puppeteer-core": "25.16.0", "vue": "^3.5.44", "zod": "^4.6.7" }, diff --git a/apps/webext/wxt.config.ts b/apps/webext/wxt.config.ts index 3f016521..7cb5451b 100644 --- a/apps/webext/wxt.config.ts +++ b/apps/webext/wxt.config.ts @@ -74,11 +74,11 @@ export default defineConfig({ 'activeTab', 'scripting', 'alarms', - ...(browser === 'firefox' ? [] : ['offscreen']), + ...(browser === 'firefox' ? [] : ['offscreen', 'debugger']), ], optional_permissions: ['bookmarks', 'history'], optional_host_permissions: ['http://*/*', 'https://*/*'], - ...(browser === 'firefox' ? {} : { minimum_chrome_version: '116' }), + ...(browser === 'firefox' ? {} : { minimum_chrome_version: '125' }), ...(browser === 'firefox' ? { browser_specific_settings: { @@ -105,6 +105,9 @@ export default defineConfig({ vite: () => ({ resolve: { alias: { + '@inkcre/core/browser-automation/protocol': fileURLToPath( + new URL('../../packages/core/src/browser-automation/protocol.ts', import.meta.url) + ), '@inkcre/core': fileURLToPath(new URL('../../packages/core/src/index.ts', import.meta.url)), }, }, diff --git a/docs/30-unit-tdd/README.md b/docs/30-unit-tdd/README.md index d610f656..8f54b76e 100644 --- a/docs/30-unit-tdd/README.md +++ b/docs/30-unit-tdd/README.md @@ -13,6 +13,8 @@ not architectural boundaries. browser runtime handles, and rendering boundaries. - [Native Extension Runtime](native-extension-runtime.md) owns the Extension Host, producer/Host boundary, native Module Federation loading, durable enabled intent, and compensated lifecycle. +- [Web Source 本机浏览器能力](browser-automation.md) owns origin pairing, exact browser binding, + Puppeteer transport, and the lifetime of borrowed or created automation tabs. Repository/package topology remains in [`../../ARCHITECTURE.md`](../../ARCHITECTURE.md). Observable product behavior and cross-unit contracts remain in the read-only diff --git a/docs/30-unit-tdd/bilibili-extension.md b/docs/30-unit-tdd/bilibili-extension.md new file mode 100644 index 00000000..2a351fe7 --- /dev/null +++ b/docs/30-unit-tdd/bilibili-extension.md @@ -0,0 +1,33 @@ +# B 站收藏夹插件 + +本文适用于 `inkcre/bilibili` Web MF Distribution 的维护者。共同产品承诺与跨 Distribution 合同由共享文档拥有；本 Unit 负责浏览器站点适配、Source 执行、图投影及读取界面。Python 只读 Distribution 解释相同 v1 记录，为既有读取和词法检索提供能力，不拥有或控制 Web Source 的执行。 + +## 所有权 + +插件拥有 B 站账号、收藏夹、视频和收藏条目的解释，站点路径与接口字段也留在插件内。通用 Core / Web-ext 只提供浏览器绑定、Puppeteer Page、消息桥和会话生命周期，不包含 B 站选择器或业务模型。Web Peer 以普通 `core.source.collect.v1` Job 执行 Source。 + +首版范围为用户自己创建且明确选定的收藏夹，含本人可访问的私密收藏夹。账号、收藏夹选择和浏览器绑定写入 Source 配置；浏览器 cookie、密码和视频二进制不会写入配置或业务图。第三方站点接口不是 InKCre 对外稳定合同，其响应由 `site.ts` / `normalize.ts` 转换。 + +## 共享图的本地实现 + +四个精确 Resolver 为 `extensions.bilibili.user.v1`、`folder.v1`、`video.v1`、`membership.v1`，完整前缀相同。领域内容校验由 `schema.ts` 执行。账号以 MID、收藏夹以完整 media_id、视频以 aid 标识；BVID 是已知视频别名，原始 fid 不能代替 media_id。 + +数据库 identity namespace 为 `inkcre/bilibili`，key 分别为 `user:{mid}`、`folder:{media_id}`、`video:{aid}`、`membership:{media_id}:{aid}`。账号到收藏夹使用 `owns`，收藏夹到成员使用 `contains`，成员到视频使用 `item`，视频到 UP 主使用 `author`。成员自己的 Block 承载可变收藏时间、观察位置与条目可用性；稳定边不把这些值串入身份。观察位置明确为 `page_number` 与 `position_in_page`，不由分页大小推算并声称原生绝对顺序。内容不重复保存跨实体关联作为第二份权威。 + +来源没有取得的字段不写入 patch，数据库顶层合并保留已有事实。失效视频的占位标题、空缺作者或简介不能抹掉已保存事实。独立视频身份不依赖某次 Source 或浏览器，重复采集和跨收藏夹采集复用相同 Block。 + +## 批次与恢复 + +`collect.ts` 按配置的收藏夹逐一读取，每批最多处理一个 20 项分页；`has_more` 是结束依据。来源可能返回不足 20 项但仍有后续，不能用条目数代替结束标记。 + +Source.state 仅保存范围、分页提示、首批和已提交边界的身份序列、进度及计数，不存平行信息内容。继续前比较首批身份、来源报告数量及已提交边界；不同则从头重新核对该收藏夹。此策略减少普通新增和移动造成的漏取，不提供实时列表的原子快照或自动删除语义。 + +末批提交前重新读取首批，对照本轮首批身份、起始报告数量与末批报告数量；发生变化时不提交“完成”，保留此前已确认的批次，并使 Job 明确失败。完整遍历与条目全数取得是不同声明：每个收藏夹同时保留累计返回、解析视频、失效、未支持、未解析数量和来源报告数，设置界面展示差额。数量不等本身不会触发无限重扫，也不能被隐去后声称完整；此边界检查不检测任意中间位置变化。 + +每批使用准入数据库 RPC `commit_source_batch`，同时写入原生身份映射、图、Source 锚点与下一进度。Source 配置和状态以 CAS 检查，避免并发采集覆盖进度。客户端只在确认提交后采用下一进度；失败和未知结果不自动重放。`Source.block` 指向该 Source 独占的 `core.source.v1` 锚点，锚点以稳定 `collects` 边指向共享账号业务根。RPC 的 `root_key` 选择业务根，回包 `source_block` 才是可赋给 Source.block 的锚点 ID；不同 Source 可采集同一个账号而保持各自来源身份。设置变更只更新配置，不顺便写业务图。 + +## 读取 + +Resolver 仅依赖已保存的 Block 和稳定关系；不会为读取而连接原浏览器或调用 B 站。收藏条目按归档关系分批读取，每次最多 50 个并提供继续入口，避免大收藏夹在单个图预览中被截断。视频所在收藏夹经成员 Block 反向解析，收藏时间与可用性跟随对应成员显示。 + +首轮仅保存元数据。原文和封面 URL 仍指向来源；文字和关系的离线使用不取决于这些链接继续可用。视频播放、字幕、源站编辑与失效内容恢复不在此 Unit 的当前承诺中。 diff --git a/docs/30-unit-tdd/browser-automation.md b/docs/30-unit-tdd/browser-automation.md new file mode 100644 index 00000000..5e940b04 --- /dev/null +++ b/docs/30-unit-tdd/browser-automation.md @@ -0,0 +1,63 @@ +# Web Source 的本机浏览器能力 + +当 Web 中运行的 extension Source 需要借用用户浏览器的登录态、页面交互或页面网络响应时，使用本 Unit。Source 继续由普通 MF Distribution 交付，Web Peer 通过共享 Job 协议领取任务；Web-ext 承担本机标签页和调试会话执行。站点业务逻辑与数据模型不进入 SDK 或 Web-ext。 + +## 资格与配置 + +`SourceImplementation.canCollect(source, config)` 与 `canBackfill` 是可选的业务资格钩子。普通 Source 没有钩子时沿用已注册实现的判断。浏览器 Source 可调用 `BrowserAccess.hasBinding(provider)` 检查本 Web Peer 是否已明确配置该兼容 Provider；不得用实时连接、权限或登录检查使任务无声停留 pending。命令配置不合法时仍进入执行验证并失败。 + +Provider 是 Web-ext 加入所选信息库时配置的确切 Peer UUID。浏览器实例另有保存在扩展本机 storage 的随机 `instance`；运行中的 service worker 再提供短期 `epoch`。绑定同时固定 Provider 和 instance，不根据另一个在线实例替换浏览器或账号。 + +Web 绑定保存在本机 localStorage，以当前 Web Peer UUID 和数据库地址分隔。`bindings()` 和 `hasBinding()` 只读这份配置；它们不声称浏览器当前在线。未配置时 Source 不领取；已有绑定但桥断开、授权撤销或账号失效时，已领取的执行明确失败。 + +## 原生授权和通信 + +1. 用户在 Web-ext 设置中为当前信息库授权可信的 InKCre Web 地址。 +2. 原生 host 权限通过明确的用户操作请求，扩展只为授权的站点注册打包内的 bridge content script。 +3. Web 调用 `BrowserAccess.discover()`，用户明确选择后 `bind(binding)`；选择结果不是自动授予扩展端权限。 +4. Web 页面通过同 origin 的 `window.postMessage` 与 content script 通信，再由 `runtime.Port` 到 background。background 对每次请求检查真实 sender、顶层 frame、完整 origin、当前原生授权及精确 Provider/instance。 + +浏览器的 host 权限模式不区分端口；扩展端的完整 origin 授权会区分。允许一个 localhost 端口不会使另一端口获得自动化权限。消息的名称、UUID 和 namespace 都不替代授权；同一已授权 origin 的页面脚本处于同一个信任范围。 + +动态授权适用于自托管地址，不需要将用户域名写入 manifest，也不需要云端或 HTTP 中继。撤销授权释放该 origin 的会话并移除后续脚本注册。已有页面上的 bridge 不能绕过 background 的实时检查。 + +## Source 接口 + +```ts +await BrowserAccess.withPage( + { provider: configuredProvider, target: { kind: 'new' }, signal }, + async (page) => { + await page.goto(sourceUrl) + await page.waitForSelector('[data-ready]') + return page.evaluate(() => document.title) + } +) +``` + +`BrowserPage` 是 Puppeteer 的真实 `Page` 类型。`tabs(provider, signal?)` 返回当前可借用的 HTTP(S) 标签页短期引用；`target: {kind: 'existing', tab}` 必须使用当前枚举结果。引用绑定调用页面、Provider、实例、service-worker epoch 和单次 reference UUID，枚举刷新或一次使用后失效；不能把裸 tabId 当成可长期持有的权限。 + +新建标签页正常结束时关闭；借用标签页解除调试并保留。开始控制时将目标设为其浏览器窗口的活动标签页，保证浏览器渲染和原生点击可执行；这会改变该窗口的活动标签页，不声称恢复用户原来的浏览位置。相同标签页的控制互斥，旧 Page 不能在结束后继续发命令。网页已发生的变化不会回滚，Source 自己负责账号确认、站点身份、分页、预算、增量进度和业务写入。 + +Abort 会中断 Page 连接、请求后台释放，并等待 Source callback 退出与原生清理；不会通过 Promise.race 提前把仍执行中的 callback 算作结束。bridge 断连向请求与 Page 传播失败。已经派发的效果可能未知，不自动重放命令或替换 Provider。打开动作的会话 ID 由调用方预先产生，打开响应丢失后仍能向原目标请求清理。 + +## Puppeteer 适配边界 + +依赖固定为 `puppeteer-core@25.16.0`，两端使用同一版本。Web-ext 复用官方实验性 `ExtensionTransport`，由 `chrome.debugger` 执行；每个连接对应一个标签页，标签页创建和释放由 Web-ext 完成。 + +25.16 的 public browser entry 在构建图中仍引入 Node-only 的 channel 连接分支。SDK 使用该固定版本的窄 CDP connector，等同于官方 connect 在给定 transport 后选择的 CDP 路径，避免要求每个 Host/MF producer 设置 Node externals。它是明确的内部入口维护点；升级依赖时需重新检查与实测。Page、Connection 和自动化命令仍来自 Puppeteer，不是自实现的兼容层。background 静态导入窄 ExtensionTransport 模块，避免 service worker 执行动态 import。 + +该版本的 `ExtensionTransport.close()` 原本丢弃 native detach Promise。本仓库仅补丁修改其返回值及声明，使外层可以等待原生 detach 完成；补丁行为测试验证等待和事件监听清理。外层另监听原生 `debugger.onDetach`，处理目标关闭、DevTools 抢占等会话中断。上游提供等价保证时移除该补丁。 + +Chromium 构建声明必选 `debugger` 权限，最低 Chrome 125，覆盖 flat child sessions；该权限不能作为 optional 权限运行时申请。Firefox 构建保留既有材料采集，但不暴露这条 debugger 自动化能力。不能从一次 Page 操作通过推定完整 Puppeteer API 兼容，也不承诺关闭 Web 或浏览器后继续 Job。 + +协议 DTO 由 `@inkcre/core/browser-automation/protocol` 独立入口交付，content script 只导入这个轻量入口，避免运行 Core root 的 Job 注册等副作用。 + +## 验证入口 + +`pnpm exec node --test scripts/tests/browser-automation-transport.test.mjs` 检查限定版本补丁的 native Promise 行为。`pnpm exec node scripts/verify-browser-automation.mjs` 使用生产扩展构建、真实 SDK 和隔离 Chromium profile 运行合成登录站点；测试进程不连接远程 CDP，因此不会占用被测标签页的调试连接。它检查实际导航、登录态、点击和后续批次、响应体、标签页清理、借用页保留、旧引用、Abort、精确 Provider 与同 host 不同 origin 的授权边界。这些检查也接入 `pnpm test:e2e:webext`，位于既有材料采集回归之前。 + +同 origin 的多个 Provider 共用 Port。撤销 A 只释放 A 的会话；B 仍获授权时不能断开共用 Port。隔离验收在 B 的真实 Page 会话活跃时撤销 A，检查 B 仍可执行。`scripts/tests/job-cancellation.test.mjs` 检查已取消 pending Job 及 claim 期间收到取消时不启动业务 handler；普通不合格任务继续 pending。 + +合成站点的测试副本只为合成 host 预授予权限，不能替代原生授权 UI 或真实 B 站登录态的业务验收。用户实际业务数据的验收由相应扩展与任务证据维护。 + +依据：[Puppeteer 扩展指南](https://pptr.dev/guides/running-puppeteer-in-extensions)、[Chrome debugger](https://developer.chrome.com/docs/extensions/reference/api/debugger)。 diff --git a/docs/30-unit-tdd/info-base.md b/docs/30-unit-tdd/info-base.md index 3c923190..981fb1fa 100644 --- a/docs/30-unit-tdd/info-base.md +++ b/docs/30-unit-tdd/info-base.md @@ -75,6 +75,12 @@ manager does not impose a universal classification ladder. ## Browser Handles and Rendering +`core.source.v1` is a provenance anchor rather than a business collection root. Its strict content +contains only Source `id`, `type`, and optional `nickname`; the native Source Resolver projects the +same readable sentence as the Python implementation and uses the existing text/preview renderers. +Opening a Source does not load credentials or run a collector. Outgoing domain relations lead to +the persisted business graph, which keeps its own exact Resolver contracts. + Byte-oriented Resolvers may expose Blob or Object URL handles for image, audio, video, PDF, EPUB, ZIP, and file content. Handles are private runtime state, never persisted solved-graph authority. They are revoked on Resolver refresh, disposal, cache replacement, and eviction. diff --git a/docs/40-deployment/development-runtime.md b/docs/40-deployment/development-runtime.md index 74618660..74fa0de8 100644 --- a/docs/40-deployment/development-runtime.md +++ b/docs/40-deployment/development-runtime.md @@ -22,13 +22,14 @@ The committed [`svc.json`](../../svc.json) declares four worktree-scoped capabilities: `database`, `web`, `web-ui`, and `webext`. SVC resolves the worktree instance and coordinates provisioning and -readiness; [`scripts/dev.mjs`](../../scripts/dev.mjs) and the database runtime scripts own the +readiness through schema 5 resources, actions and observations; [`scripts/dev.mjs`](../../scripts/dev.mjs) and the database runtime scripts own the executable behavior. Observe without provisioning with `pnpm dev:status`, and run `pnpm run doctor` before diagnosing host setup. Portless gives the web and WXT targets instance-specific HTTPS `.localhost` routes. Their probes -accept only the expected target and worktree identity. Provisioning returns after readiness; it does -not transfer long-lived process ownership to SVC. The optional `web-ui` target is a non-release +accept only the expected target and worktree identity. Database provisioning completes on exit and +the ready observation; long-running web and WXT actions complete on readiness and retain their +managed process owner. The optional `web-ui` target is a non-release source lane whose resolved checkout stays machine-local. Normal build, check, and CI continue to use the locked Registry package. @@ -153,6 +154,20 @@ Pull-request evidence does not establish source-branch or migration equality bet 独占的 typegen 数据库，不接受可移动 tag，也不会拉取本地 ID。此入口不支持 external provider；需选择构建候选的 同一 provider。`contracts/core-release.json` 仍只接纳已发布的 GHCR digest，本地生成不能代替发布产物验收。 +联合开发尚未发布的数据库协议时，也可以从已就绪的 Python 工作树 runtime 生成类型，不拉取稳定镜像或创建第二个数据库： + +```bash +# 在拥有该 runtime 的 impl-py 工作树导出当前协议。 +pdm run db:contract > .runtime/current-contract.json + +# 在 impl-ts 工作树显式指定 owner descriptor 与刚导出的合同。 +pnpm contract:sync --runtime /absolute/impl-py/.runtime/database/INSTANCE/runtime.json --runtime-contract /absolute/impl-py/.runtime/current-contract.json +``` + +此入口只接受 `InKCre/impl-py` 的 development descriptor，合同 revision 与 migration head 必须匹配。 +它通过 descriptor 指定的数据库 owner 运行 Supabase CLI，读取本机 credential 文件但不输出连接凭据；不停止或修改 owner runtime。 +追加 `--check` 验证生成结果。未传这两个参数时仍使用已发布镜像路径；本地生成不会修改 canonical candidate，也不能作为正式发布证据。 + ## Operator Commands Run these from the repository root; [`package.json`](../../package.json) owns their exact carriers. diff --git a/docs/40-deployment/native-extension-delivery.md b/docs/40-deployment/native-extension-delivery.md index 248021b1..1b2db40d 100644 --- a/docs/40-deployment/native-extension-delivery.md +++ b/docs/40-deployment/native-extension-delivery.md @@ -71,11 +71,14 @@ copy so the preview can load a mutually compatible Host and MF when a change rai Host SDK. This produces temporary derived versions for preview only. It neither consumes the pending changesets in the PR nor replaces the normal Version PR and protected-main release lifecycle. -The Workspace contract check applies the same preparation after `pnpm check` has validated the -original checkout. It builds the isolated candidate Host and MF before inspecting the native -snapshot, and records the exact checked-out commit SHA in the job summary. For pull requests that -SHA can be GitHub's synthetic merge commit; it identifies the source actually exported by -`git archive`, while static, typed, and package checks still run on the original checkout first. +Workspace contract 先在原 checkout 校验格式、静态规则和类型，再通过 +`scripts/prepare-native-checkout.mjs` 将当前未忽略的源文件复制到隔离候选，运行正式 +`pnpm release:version`，在候选中完成生产构建、资产闭包和发布检查。这样新增 API 的 MF +可以明确要求下一兼容 SDK，而无需放宽为兼容旧已发布 SDK；原版本和 Changesets 保持不变。 + +本地可运行 `node scripts/prepare-native-checkout.mjs --output .runtime/native-checkout --offline`， +随后进入输出的 `workspace` 执行 `pnpm build`。输出目录必须为空；准备过程记录原 HEAD、 +是否有未提交变更与源快照摘要。它只为验证派生版本，不发布资产或创建提交。 A plugin owns compatibility of its shared configuration, state, Source/Job parameters and stored information. Changesets record MF changes even when no Python change is necessary. Published native diff --git a/extensions/bilibili/CHANGELOG.md b/extensions/bilibili/CHANGELOG.md new file mode 100644 index 00000000..2392e92b --- /dev/null +++ b/extensions/bilibili/CHANGELOG.md @@ -0,0 +1,7 @@ +# Bilibili Extension + +## 0.1.0 + +- 新增自己的公开及私密收藏夹采集、原生身份与成员关系存储，以及独立读取视图。 +- 通过所选浏览器的登录态执行普通 collection Job，支持有界分页和经边界复核的继续采集。 +- 保存文字与元数据，不保存视频二进制。 diff --git a/extensions/bilibili/LICENSE b/extensions/bilibili/LICENSE new file mode 100644 index 00000000..94e962af --- /dev/null +++ b/extensions/bilibili/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 InKCre contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/extensions/bilibili/README.md b/extensions/bilibili/README.md new file mode 100644 index 00000000..9af1bce5 --- /dev/null +++ b/extensions/bilibili/README.md @@ -0,0 +1,42 @@ +# B 站收藏夹 + +`inkcre/bilibili` 的 Module Federation Distribution，为 Web Peer 提供自己的 B 站收藏夹采集与读取能力。采集通过用户明确选择的浏览器执行，沿用其中的 B 站登录态；资料保存到共享信息库后，其他加载此插件的 Web Peer 可以独立读取。 + +## 使用 + +1. 在 InKCre 浏览器扩展设置中授权当前 Web 地址，并在同一浏览器登录 B 站。 +2. 打开插件设置，查找并选择该浏览器，读取 B 站收藏夹。 +3. 确认显示的账号，选择一个或多个自己的收藏夹并保存采集源。 +4. 开始采集。任务使用普通 Source / Job 路径，由具备此 Source 实现和对应浏览器绑定的 Web Peer 领取。 +5. 查看采集状态和已保存资料。单次分页预算用完时，任务可以正常结束，但设置会明确显示仍有未完成范围；再次执行会校验边界后继续。 + +MF Distribution 提供采集与独立读取能力。Python 的只读 Distribution 解释相同 v1 图合同，供既有读取与词法检索使用；它不是采集控制者，采集不要求运行 Python 业务进程。设置、账号确认或查找浏览器不会自动创建采集任务。运行期间应保持 Web Peer 与所选浏览器在线。任务运行时可以请求取消，界面会等待实际终态，不将请求发出等同于已经停止。 + +## 保存范围 + +- 自己创建且明确选择的收藏夹，包括私密收藏夹。 +- 收藏夹、视频、UP 主的原生身份和实际取得的标题、简介、链接等资料。 +- 收藏关系，以及来源明确返回的收藏时间、位置和条目可用性。同一视频位于多个收藏夹时共用视频实体。 +- 多 P 视频仍以被收藏的视频对象为单位。 + +本版不保存视频二进制，不下载字幕或评论，也不修改 B 站收藏。封面仅保存来源 URL；来源图片失效不会影响已保存的文字与关系。私密收藏夹的资料同样进入当前信息库，请按自己的信息库访问范围选择。 + +## 完整性与重复执行 + +分页以来源的 `has_more` 为准，短页不能被当成末尾。恢复前复核列表首批、数量和已提交分页边界；观察到变化时重新扫描。末批提交前再次核对首批和报告数量，发现变化则保留已确认批次并明确失败，供下一次重新核对。页码只是可复核的执行提示，不是永久信息身份。 + +“分页已遍历”仅表示可见分页到达结束标记并通过上述边界检查。界面同时显示累计返回条目、来源报告数量和差额；来源可能隐藏部分条目或异步更新计数，差额不能被当作已成功取得。列表在采集期间变化时，本版不承诺某一时刻的原子快照，也不声称能检测任意中间位置的重排。 + +每批图变化和进度通过数据库 `commit_source_batch` 同事务确认。未知写入结果不会自动重放。再次采集复用原生身份和既有 Block，保留用户添加的关系；本版不会根据未出现的条目删除历史收藏资料。 + +失效占位标题不会覆盖已有标题或简介。非视频和身份不足条目计入未支持或未解析数量。未登录、账号变化、挑战响应、分页异常或断连会使任务明确失败，不能被当成空收藏夹成功。 + +## 开发与验证 + +```sh +pnpm --filter @inkcre/ext-bilibili test +pnpm --filter @inkcre/ext-bilibili type-check +pnpm --filter @inkcre/ext-bilibili build +``` + +单元回归使用虚构身份和文本，只复用实际观察到的响应字段形状；不在仓库保存用户私人收藏响应。真实浏览器链和私密收藏夹验收由主任务记录单独报告。 diff --git a/extensions/bilibili/index.html b/extensions/bilibili/index.html new file mode 100644 index 00000000..423310ef --- /dev/null +++ b/extensions/bilibili/index.html @@ -0,0 +1,12 @@ + + + + + + Bilibili Extension + + +
+ + + diff --git a/extensions/bilibili/package.json b/extensions/bilibili/package.json new file mode 100644 index 00000000..ba0e3859 --- /dev/null +++ b/extensions/bilibili/package.json @@ -0,0 +1,48 @@ +{ + "name": "@inkcre/ext-bilibili", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "type-check": "vue-tsc --noEmit", + "test": "node --experimental-strip-types --test tests/*.test.ts" + }, + "dependencies": { + "@inkcre/core": "workspace:*" + }, + "devDependencies": { + "@inkcre/ext-dev-utils": "workspace:*", + "@inkcre/ui-web": "2.7.0", + "@module-federation/vite": "^1.1.0", + "@vitejs/plugin-vue": "^6.0.1", + "@vue/tsconfig": "catalog:", + "sass-embedded": "^1.90.0", + "typescript": "catalog:", + "vite": "^7.0.6", + "vue-tsc": "catalog:", + "zod": "^4.6.7" + }, + "peerDependencies": { + "@vueuse/core": "^14.0.0", + "pinia": "^3.0.0", + "vue": "^3.5.25", + "vue-router": "^4.5.0", + "zod": "^4.1.0" + }, + "inkcre": { + "name": "inkcre/bilibili", + "nickname": "哔哩哔哩", + "module_federation": { + "host_sdk": "@inkcre/core", + "host_sdk_version": ">=0.4.4 <0.5.0" + } + }, + "intent": { + "skills": [ + "@inkcre/ui-web" + ] + } +} diff --git a/extensions/bilibili/src/BilibiliContent.vue b/extensions/bilibili/src/BilibiliContent.vue new file mode 100644 index 00000000..3b41e9c4 --- /dev/null +++ b/extensions/bilibili/src/BilibiliContent.vue @@ -0,0 +1,175 @@ + + + + + diff --git a/extensions/bilibili/src/BilibiliPreview.vue b/extensions/bilibili/src/BilibiliPreview.vue new file mode 100644 index 00000000..5922dd9c --- /dev/null +++ b/extensions/bilibili/src/BilibiliPreview.vue @@ -0,0 +1,50 @@ + + + + + diff --git a/extensions/bilibili/src/BilibiliSetup.vue b/extensions/bilibili/src/BilibiliSetup.vue new file mode 100644 index 00000000..79e311b7 --- /dev/null +++ b/extensions/bilibili/src/BilibiliSetup.vue @@ -0,0 +1,600 @@ + + + + + diff --git a/extensions/bilibili/src/DevApp.vue b/extensions/bilibili/src/DevApp.vue new file mode 100644 index 00000000..543a743a --- /dev/null +++ b/extensions/bilibili/src/DevApp.vue @@ -0,0 +1,8 @@ + + + diff --git a/extensions/bilibili/src/collect.ts b/extensions/bilibili/src/collect.ts new file mode 100644 index 00000000..5dcf773f --- /dev/null +++ b/extensions/bilibili/src/collect.ts @@ -0,0 +1,63 @@ +import type { FavoritePage } from './normalize.ts' +import type { BilibiliSourceConfig, CollectConfig, User } from './schema.ts' +import { + advanceProgress, + canResume, + emptyCursor, + verifyCompletion, + type Progress, +} from './progress.ts' + +export interface FavoriteSessionPort { + open(folder: string): Promise + account(): Promise + read(folder: string, page: number): Promise +} + +export async function collectFavorites( + config: BilibiliSourceConfig, + initial: Progress, + options: CollectConfig, + signal: AbortSignal, + session: FavoriteSessionPort, + commit: (account: User, page: FavoritePage, progress: Progress) => Promise +): Promise { + let progress = initial + for (const folder of config.folder_ids) { + signal.throwIfAborted() + let cursor = progress.folders[folder] ?? emptyCursor() + if (cursor.complete) continue + await session.open(folder) + await session.account() + let cachedHead: FavoritePage | null = null + if (cursor.next_page > 1) { + const head = await session.read(folder, 1) + const boundary = + cursor.next_page === 2 ? head : await session.read(folder, cursor.next_page - 1) + if (!canResume(cursor, head, boundary)) { + cursor = emptyCursor() + progress = { ...progress, folders: { ...progress.folders, [folder]: cursor } } + cachedHead = head + } + } + while (!cursor.complete && progress.last_run.pages < options.max_pages) { + signal.throwIfAborted() + const account = await session.account() + const current = cachedHead ?? (await session.read(folder, cursor.next_page)) + cachedHead = null + signal.throwIfAborted() + if (!current.hasMore) { + const finalHead = await session.read(folder, 1) + signal.throwIfAborted() + verifyCompletion(cursor, current, cursor.next_page, finalHead) + } + const next = advanceProgress(progress, folder, current, cursor.next_page, options.max_pages) + await commit(account, current, next) + // Progress is adopted only after the database confirms the graph and cursor together. + progress = next + cursor = progress.folders[folder]! + } + if (progress.last_run.pages >= options.max_pages) break + } + return progress +} diff --git a/extensions/bilibili/src/graph.ts b/extensions/bilibili/src/graph.ts new file mode 100644 index 00000000..24ea0160 --- /dev/null +++ b/extensions/bilibili/src/graph.ts @@ -0,0 +1,50 @@ +import { resolverTypes, type User } from './schema.ts' +import type { FavoritePage } from './normalize.ts' + +export interface GraphRecord { + key: string + resolver: string + content: Record +} +export interface GraphRelation { + from_key: string + to_key: string + content: string +} +export interface SourceGraphBatch { + records: GraphRecord[] + relations: GraphRelation[] +} + +function knownFields(value: object): Record { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) +} + +export function favoriteGraph(account: User, page: FavoritePage): SourceGraphBatch { + const records = new Map() + const relations = new Map() + function record(key: string, resolver: string, value: object) { + const existing = records.get(key) + records.set(key, { key, resolver, content: { ...existing?.content, ...knownFields(value) } }) + } + function relate(from_key: string, to_key: string, content: string) { + relations.set(JSON.stringify([from_key, to_key, content]), { from_key, to_key, content }) + } + record(`user:${account.mid}`, resolverTypes.user, account) + record(`user:${page.owner.mid}`, resolverTypes.user, page.owner) + record(`folder:${page.folder.id}`, resolverTypes.folder, page.folder) + relate(`user:${account.mid}`, `folder:${page.folder.id}`, 'owns') + for (const item of page.items) { + const video = `video:${item.video.aid}` + const membership = `membership:${page.folder.id}:${item.video.aid}` + record(video, resolverTypes.video, item.video) + record(membership, resolverTypes.membership, item.membership) + relate(`folder:${page.folder.id}`, membership, 'contains') + relate(membership, video, 'item') + if (item.uploader) { + record(`user:${item.uploader.mid}`, resolverTypes.user, item.uploader) + relate(video, `user:${item.uploader.mid}`, 'author') + } + } + return { records: [...records.values()], relations: [...relations.values()] } +} diff --git a/extensions/bilibili/src/index.ts b/extensions/bilibili/src/index.ts new file mode 100644 index 00000000..0dda5b3f --- /dev/null +++ b/extensions/bilibili/src/index.ts @@ -0,0 +1,31 @@ +import { z } from 'zod' +import { Resolver, SourceManager, SourceType, type ExtensionModule } from '@inkcre/core' +import { UserResolver, FolderResolver, VideoResolver, MembershipResolver } from './resolver.ts' +import { SourceConfigSchema, CollectConfigSchema, sourceType } from './schema.ts' +import { bilibiliSource } from './source.ts' +import BilibiliSetup from './BilibiliSetup.vue' + +const resolvers = [UserResolver, FolderResolver, VideoResolver, MembershipResolver] +const extension: ExtensionModule = { + setup: { component: BilibiliSetup }, + async initialize() { + for (const resolver of resolvers) Resolver.register(resolver.type, resolver) + SourceManager.register(sourceType, bilibiliSource) + await SourceType.dbApi + .upsert({ + id: sourceType, + description: '采集自己的 B 站收藏夹，包括所选私密收藏夹。', + config_schema: z.toJSONSchema(SourceConfigSchema), + collect_config_schema: z.toJSONSchema(CollectConfigSchema), + backfill_config_schema: null, + }) + .throwOnError() + }, + async activate() {}, + async deactivate() {}, + async dispose() { + SourceManager.unregister(sourceType, bilibiliSource) + for (const resolver of resolvers) Resolver.unregister(resolver.type, resolver) + }, +} +export default extension diff --git a/extensions/bilibili/src/main.ts b/extensions/bilibili/src/main.ts new file mode 100644 index 00000000..a81fb0bc --- /dev/null +++ b/extensions/bilibili/src/main.ts @@ -0,0 +1,8 @@ +import { bootstrapExtension } from '@inkcre/ext-dev-utils' +import '@inkcre/ui-web/styles' +import App from './DevApp.vue' +import extensionModule from './index.ts' + +void bootstrapExtension({ rootComponent: App, extensionModule }).catch(() => + console.error('无法启动 B 站插件开发页面。') +) diff --git a/extensions/bilibili/src/normalize.ts b/extensions/bilibili/src/normalize.ts new file mode 100644 index 00000000..e1951773 --- /dev/null +++ b/extensions/bilibili/src/normalize.ts @@ -0,0 +1,200 @@ +import { z } from 'zod' +import { + FolderSchema, + MembershipSchema, + UserSchema, + VideoSchema, + nativeId, + type Folder, + type Membership, + type User, + type Video, +} from './schema.ts' + +const object = z.record(z.string(), z.unknown()) +export interface FavoriteItem { + video: Video + uploader: User | null + membership: Membership +} +export interface FavoritePage { + folder: Folder + owner: User + items: FavoriteItem[] + hasMore: boolean + unsupported: number + unresolved: number + totalItems: number + ids: string[] +} + +export function responseData(response: unknown): Record { + const envelope = object.parse(response) + if (envelope.code !== 0) { + const message = typeof envelope.message === 'string' ? envelope.message : 'B 站未返回成功结果。' + throw new Error(`B 站请求失败（${String(envelope.code)}）：${message}`) + } + return object.parse(envelope.data) +} + +function text(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined +} +function seconds(value: unknown): number | undefined { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined +} +function instant(value: unknown): string | undefined { + const timestamp = seconds(value) + if (!timestamp) return undefined + const date = new Date(timestamp * 1000) + return Number.isNaN(date.getTime()) ? undefined : date.toISOString() +} +function webUrl(value: unknown): string | undefined { + if (typeof value !== 'string' || !value) return undefined + try { + const url = new URL(value.startsWith('//') ? `https:${value}` : value) + return ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password + ? url.href + : undefined + } catch { + return undefined + } +} + +export function parseUser(value: unknown): User { + const user = object.parse(value) + return UserSchema.parse({ + mid: nativeId(user.mid), + name: text(user.name ?? user.uname), + avatar_url: webUrl(user.face), + }) +} + +export function parseAccount(response: unknown): User { + const data = responseData(response) + if (data.isLogin !== true) throw new Error('请先在所选浏览器登录 B 站，再读取收藏夹。') + return parseUser(data) +} + +export function parseFolder( + value: unknown, + expectedOwner: string +): { folder: Folder; owner: User } { + const info = object.parse(value) + const owner = parseUser(info.upper ?? { mid: info.mid }) + if (owner.mid !== expectedOwner || (info.mid != null && nativeId(info.mid) !== expectedOwner)) + throw new Error('收藏夹不属于已确认的 B 站账号。') + return { + owner, + folder: FolderSchema.parse({ + // info.fid is the original folder number; info.id is the full media_id used by the site. + id: nativeId(info.id), + title: text(info.title), + description: text(info.intro), + is_private: typeof info.attr === 'number' ? (info.attr & 1) === 1 : undefined, + reported_count: seconds(info.media_count), + }), + } +} + +export function parseFolders(response: unknown, owner: string): Folder[] { + const data = responseData(response) + if (!Array.isArray(data.list)) throw new Error('B 站未返回可识别的自建收藏夹列表。') + return data.list.map((value) => { + const row = object.parse(value) + return parseFolder({ ...row, upper: row.upper ?? { mid: row.mid ?? owner } }, owner).folder + }) +} + +export function parseFavoritePage( + response: unknown, + expectedOwner: string, + expectedFolder: string, + page: number, + observedAt = new Date().toISOString(), + pageSize = 20 +): FavoritePage { + const data = responseData(response) + const { folder, owner } = parseFolder(data.info, expectedOwner) + if (folder.id !== expectedFolder) throw new Error('B 站返回了另一个收藏夹的数据。') + if (typeof data.has_more !== 'boolean') + throw new Error('B 站分页结束标记缺失，无法确认采集范围。') + const medias = data.medias === null && !data.has_more ? [] : data.medias + if (!Array.isArray(medias)) throw new Error('B 站未返回可识别的收藏条目。') + if (medias.length > pageSize) + throw new Error('B 站返回了超出分页范围的条目，需重新核实分页协议。') + if (medias.length === 0 && data.has_more) + throw new Error('收藏分页为空但仍声明有后续，已停止采集。') + const items: FavoriteItem[] = [] + let unsupported = 0 + let unresolved = 0 + for (let index = 0; index < medias.length; index++) { + const raw = object.safeParse(medias[index]) + if (!raw.success) { + unresolved++ + continue + } + const value = raw.data + if (value.type !== 2) { + unsupported++ + continue + } + try { + const aid = nativeId(value.id) + const unavailable = + value.title === '已失效视频' || (typeof value.attr === 'number' && (value.attr & 1) === 1) + const bvid = z + .string() + .regex(/^BV[0-9A-Za-z]{10}$/) + .safeParse(value.bvid ?? value.bv_id) + let uploader: User | null = null + if (value.upper != null) { + try { + uploader = parseUser(value.upper) + } catch { + /* A missing uploader must not discard an identified video. */ + } + } + items.push({ + video: VideoSchema.parse({ + aid, + bvid: bvid.success ? bvid.data : undefined, + title: unavailable ? undefined : text(value.title), + description: unavailable ? undefined : text(value.intro), + duration_seconds: unavailable ? undefined : seconds(value.duration), + published_at: unavailable ? undefined : instant(value.pubtime), + cover_url: unavailable ? undefined : webUrl(value.cover), + }), + uploader, + membership: MembershipSchema.parse({ + availability: unavailable ? 'unavailable' : 'available', + page_number: page, + position_in_page: index, + favored_at: instant(value.fav_time), + observed_at: observedAt, + }), + }) + } catch { + unresolved++ + } + } + const ids = medias.map((value, index) => { + const row = object.safeParse(value) + if (!row.success) return `unresolved:${index}` + try { + return `${String(row.data.type)}:${nativeId(row.data.id)}` + } catch { + return `unresolved:${index}` + } + }) + return { + folder, + owner, + items, + hasMore: data.has_more, + unsupported, + unresolved, + totalItems: medias.length, + ids, + } +} diff --git a/extensions/bilibili/src/progress.ts b/extensions/bilibili/src/progress.ts new file mode 100644 index 00000000..36626636 --- /dev/null +++ b/extensions/bilibili/src/progress.ts @@ -0,0 +1,163 @@ +import { z } from 'zod' +import { SourceConfigSchema, type BilibiliSourceConfig } from './schema.ts' +import type { FavoritePage } from './normalize.ts' + +const CursorSchema = z.strictObject({ + next_page: z.number().int().positive(), + head_ids: z.array(z.string()).max(20), + boundary_ids: z.array(z.string()).max(20), + reported_count: z.number().int().nonnegative().optional(), + complete: z.boolean(), + rows: z.number().int().nonnegative(), + items: z.number().int().nonnegative().default(0), + unavailable: z.number().int().nonnegative().default(0), + unsupported: z.number().int().nonnegative().default(0), + unresolved: z.number().int().nonnegative().default(0), +}) +export const ProgressSchema = z.strictObject({ + version: z.literal(1), + scope: SourceConfigSchema, + folders: z.record(z.string(), CursorSchema), + last_run: z.strictObject({ + started_at: z.iso.datetime(), + updated_at: z.iso.datetime(), + pages: z.number().int().nonnegative(), + items: z.number().int().nonnegative(), + unavailable: z.number().int().nonnegative(), + unsupported: z.number().int().nonnegative(), + unresolved: z.number().int().nonnegative(), + status: z.enum(['running', 'partial', 'complete']), + }), +}) +export type Cursor = z.infer +export type Progress = z.infer + +export function emptyCursor(): Cursor { + return { + next_page: 1, + head_ids: [], + boundary_ids: [], + complete: false, + rows: 0, + items: 0, + unavailable: 0, + unsupported: 0, + unresolved: 0, + } +} + +export function scopeKey(config: BilibiliSourceConfig): string { + return JSON.stringify({ ...config, folder_ids: [...config.folder_ids].sort() }) +} + +export function startProgress( + state: Record, + config: BilibiliSourceConfig +): Progress { + const previous = ProgressSchema.safeParse(state.bilibili) + const sameScope = previous.success && scopeKey(previous.data.scope) === scopeKey(config) + const resume = sameScope && config.folder_ids.some((id) => !previous.data.folders[id]?.complete) + const now = new Date().toISOString() + return { + version: 1, + scope: config, + folders: Object.fromEntries( + config.folder_ids.map((id) => [ + id, + resume ? (previous.data.folders[id] ?? emptyCursor()) : emptyCursor(), + ]) + ), + last_run: { + started_at: now, + updated_at: now, + pages: 0, + items: 0, + unavailable: 0, + unsupported: 0, + unresolved: 0, + status: 'running', + }, + } +} + +export function sameIds(left: string[], right: string[]): boolean { + return JSON.stringify(left) === JSON.stringify(right) +} + +export function canResume(cursor: Cursor, head: FavoritePage, boundary: FavoritePage): boolean { + return ( + sameIds(cursor.head_ids, head.ids) && + cursor.reported_count === head.folder.reported_count && + sameIds(cursor.boundary_ids, boundary.ids) && + boundary.hasMore + ) +} + +export function verifyCompletion( + cursor: Cursor, + page: FavoritePage, + pageNumber: number, + head: FavoritePage +): void { + const expectedHead = pageNumber === 1 ? page.ids : cursor.head_ids + const reportedCount = pageNumber === 1 ? page.folder.reported_count : cursor.reported_count + if ( + !sameIds(expectedHead, head.ids) || + reportedCount !== head.folder.reported_count || + reportedCount !== page.folder.reported_count || + (pageNumber === 1 && head.hasMore !== page.hasMore) + ) { + throw new Error( + '收藏夹在采集期间发生变化，无法确认本轮分页已遍历；已确认的批次保留，请再次执行以重新核对。' + ) + } +} + +export function advanceProgress( + progress: Progress, + folder: string, + page: FavoritePage, + pageNumber: number, + maxPages: number +): Progress { + const previous = progress.folders[folder] ?? emptyCursor() + if (pageNumber > 1 && page.hasMore && sameIds(previous.boundary_ids, page.ids)) + throw new Error('B 站分页没有前进，已停止以避免将重复页误记为完整采集。') + if (pageNumber === 1 && page.totalItems === 0 && (page.folder.reported_count ?? 0) > 0) + throw new Error('收藏夹声明有条目但未返回内容，无法确认是空收藏夹。') + const folders = { + ...progress.folders, + [folder]: { + next_page: page.hasMore ? pageNumber + 1 : pageNumber, + head_ids: pageNumber === 1 ? page.ids : previous.head_ids, + boundary_ids: page.ids, + reported_count: pageNumber === 1 ? page.folder.reported_count : previous.reported_count, + complete: !page.hasMore, + rows: previous.rows + page.totalItems, + items: previous.items + page.items.length, + unavailable: + previous.unavailable + + page.items.filter((item) => item.membership.availability === 'unavailable').length, + unsupported: previous.unsupported + page.unsupported, + unresolved: previous.unresolved + page.unresolved, + }, + } + const pages = progress.last_run.pages + 1 + const complete = progress.scope.folder_ids.every((id) => folders[id]?.complete) + return { + ...progress, + folders, + last_run: { + ...progress.last_run, + updated_at: new Date().toISOString(), + pages, + items: progress.last_run.items + page.items.length, + unavailable: + progress.last_run.unavailable + + page.items.filter((item) => item.membership.availability === 'unavailable').length, + unsupported: progress.last_run.unsupported + page.unsupported, + unresolved: progress.last_run.unresolved + page.unresolved, + status: complete ? 'complete' : pages >= maxPages ? 'partial' : 'running', + }, + } +} diff --git a/extensions/bilibili/src/repository.ts b/extensions/bilibili/src/repository.ts new file mode 100644 index 00000000..498f0200 --- /dev/null +++ b/extensions/bilibili/src/repository.ts @@ -0,0 +1,37 @@ +import { Source, type Json } from '@inkcre/core' +import { z } from 'zod' +import { extensionName } from './schema.ts' +import type { SourceGraphBatch } from './graph.ts' +import type { Progress } from './progress.ts' + +const committedBatch = z.object({ + blocks: z.record(z.string(), z.number().int().positive()), + state: z.record(z.string(), z.unknown()), + source_block: z.number().int().positive().nullable(), +}) + +function json(value: unknown): Json { + const encoded = JSON.stringify(value) + if (encoded === undefined) throw new TypeError('采集批次需要可序列化的 JSON。') + return z.json().parse(JSON.parse(encoded)) +} + +export async function commitBatch(source: Source, graph: SourceGraphBatch, progress: Progress) { + const result = await Source.dbApi + .rpc('commit_source_batch', { + source_id: source.id, + namespace: extensionName, + expected_config: json(source.config), + expected_state: json(source.state), + root_key: `user:${progress.scope.account_mid}`, + next_state: json({ ...source.state, bilibili: progress }), + records: json(graph.records), + relations: json(graph.relations), + }) + .throwOnError() + // An ambiguous transport failure is deliberately not retried here. + const committed = committedBatch.parse(result.data) + source.state = committed.state + source.block = committed.source_block + return committed +} diff --git a/extensions/bilibili/src/resolver.ts b/extensions/bilibili/src/resolver.ts new file mode 100644 index 00000000..049b43e3 --- /dev/null +++ b/extensions/bilibili/src/resolver.ts @@ -0,0 +1,149 @@ +import { markRaw } from 'vue' +import { Block, Relation, Resolver, type ProjectionOptions } from '@inkcre/core' +import { + parseRecord, + recordTitle, + resolverTypes, + type BilibiliRecord, + type Membership, +} from './schema.ts' +import BilibiliContent from './BilibiliContent.vue' +import BilibiliPreview from './BilibiliPreview.vue' + +export type RelatedRecord = BilibiliRecord & { blockId: number; membership?: Membership } +export interface RelatedGroup { + title: string + role: 'folders' | 'videos' | 'author' | 'item' + items: RelatedRecord[] + nextOffset: number | null +} +export interface SolvedBilibili { + value: BilibiliRecord + groups: RelatedGroup[] +} + +async function recordBlock(block: Block): Promise { + const content = await block.getHydratedContent() + if (typeof content !== 'string') throw new Error('B 站资料应为文本记录。') + const value = parseRecord(block.resolver, JSON.parse(content)) + return value ? { ...value, blockId: block.id } : null +} + +async function edges(ids: number[], role: string, incoming = false, offset = 0, limit = 51) { + if (!ids.length) return [] + const result = await Relation.dbApi + .from() + .select() + .in(incoming ? 'to_' : 'from_', ids) + .eq('content', role) + .order('id', { ascending: true }) + .range(offset, offset + limit - 1) + .throwOnError() + return (result.data ?? []).map((row) => Relation.parse(row)) +} + +async function records(ids: number[]) { + const result = await Promise.all((await Block.getMany(ids)).map(recordBlock)) + return result.filter((value): value is RelatedRecord => value !== null) +} + +export abstract class BilibiliResolver extends Resolver { + static readonly previewRenderer = markRaw(BilibiliPreview) + static readonly solvedContentRenderer = markRaw(BilibiliContent) + + async readGroup(role: RelatedGroup['role'], offset = 0): Promise { + const value = parseRecord(this.block.resolver, JSON.parse(await this.getRawContent())) + if (!value) throw new Error('无法识别 B 站资料类型。') + let initial: Relation[] = [] + let title = '' + let items: RelatedRecord[] = [] + if (value.kind === 'user' && role === 'folders') { + title = '已保存的收藏夹' + initial = await edges([this.block.id], 'owns', false, offset) + items = (await records(initial.slice(0, 50).map((edge) => edge.to_))).filter( + (item) => item.kind === 'folder' + ) + } else if (value.kind === 'folder' && role === 'videos') { + title = '已保存的收藏条目' + initial = await edges([this.block.id], 'contains', false, offset) + const memberships = (await records(initial.slice(0, 50).map((edge) => edge.to_))).filter( + (item) => item.kind === 'membership' + ) + const links = await edges( + memberships.map((item) => item.blockId), + 'item', + false, + 0, + 100 + ) + const videos = await records(links.map((edge) => edge.to_)) + for (const membership of memberships) { + const link = links.find((edge) => edge.from_ === membership.blockId) + const video = videos.find((item) => item.blockId === link?.to_) + if (video?.kind === 'video' && membership.kind === 'membership') + items.push({ ...video, membership: membership.record }) + } + } else if (value.kind === 'video' && role === 'author') { + title = 'UP 主' + initial = await edges([this.block.id], 'author', false, offset) + items = (await records(initial.slice(0, 50).map((edge) => edge.to_))).filter( + (item) => item.kind === 'user' + ) + } else if (value.kind === 'video' && role === 'folders') { + title = '所在收藏夹' + initial = await edges([this.block.id], 'item', true, offset) + const membershipIds = initial.slice(0, 50).map((edge) => edge.from_) + const memberships = await records(membershipIds) + const links = await edges(membershipIds, 'contains', true, 0, 100) + const folders = await records(links.map((edge) => edge.from_)) + for (const membership of memberships) { + const link = links.find((edge) => edge.to_ === membership.blockId) + const folder = folders.find((item) => item.blockId === link?.from_) + if (folder?.kind === 'folder' && membership.kind === 'membership') + items.push({ ...folder, membership: membership.record }) + } + } else if (value.kind === 'membership' && role === 'item') { + title = '视频' + initial = await edges([this.block.id], 'item', false, offset) + items = (await records(initial.slice(0, 50).map((edge) => edge.to_))).filter( + (item) => item.kind === 'video' + ) + } else throw new Error('此资料没有所选关系。') + return { title, role, items, nextOffset: initial.length > 50 ? offset + 50 : null } + } + + protected async _getSolvedContent(options: ProjectionOptions): Promise { + const value = parseRecord(this.block.resolver, JSON.parse(await this.getRawContent(options))) + if (!value) throw new Error('无法识别 B 站资料类型。') + const roles: RelatedGroup['role'][] = + value.kind === 'user' + ? ['folders'] + : value.kind === 'folder' + ? ['videos'] + : value.kind === 'video' + ? ['author', 'folders'] + : ['item'] + return { value, groups: await Promise.all(roles.map((role) => this.readGroup(role))) } + } + + async getText(options: ProjectionOptions = {}): Promise { + const value = parseRecord(this.block.resolver, JSON.parse(await this.getRawContent(options))) + if (!value) throw new Error('无法识别 B 站资料类型。') + const description = + value.kind === 'video' || value.kind === 'folder' ? value.record.description : undefined + return [recordTitle(value), description].filter(Boolean).join('\n') + } +} + +export class UserResolver extends BilibiliResolver { + static readonly type = resolverTypes.user +} +export class FolderResolver extends BilibiliResolver { + static readonly type = resolverTypes.folder +} +export class VideoResolver extends BilibiliResolver { + static readonly type = resolverTypes.video +} +export class MembershipResolver extends BilibiliResolver { + static readonly type = resolverTypes.membership +} diff --git a/extensions/bilibili/src/schema.ts b/extensions/bilibili/src/schema.ts new file mode 100644 index 00000000..e3591ad7 --- /dev/null +++ b/extensions/bilibili/src/schema.ts @@ -0,0 +1,121 @@ +import { z } from 'zod' + +export const extensionName = 'inkcre/bilibili' +export const sourceType = 'extensions.bilibili.favorites.v1' +export const resolverTypes = { + user: 'extensions.bilibili.user.v1', + folder: 'extensions.bilibili.folder.v1', + video: 'extensions.bilibili.video.v1', + membership: 'extensions.bilibili.membership.v1', +} as const + +export const NativeIdSchema = z.string().regex(/^[1-9]\d*$/) +const instant = z.iso.datetime() +export const UserSchema = z.strictObject({ + mid: NativeIdSchema, + name: z.string().optional(), + avatar_url: z.url().optional(), +}) +export const FolderSchema = z.strictObject({ + id: NativeIdSchema, + title: z.string().optional(), + description: z.string().optional(), + is_private: z.boolean().optional(), + reported_count: z.number().int().nonnegative().optional(), +}) +export const VideoSchema = z.strictObject({ + aid: NativeIdSchema, + bvid: z + .string() + .regex(/^BV[0-9A-Za-z]{10}$/) + .optional(), + title: z.string().optional(), + description: z.string().optional(), + duration_seconds: z.number().int().nonnegative().optional(), + published_at: instant.optional(), + cover_url: z.url().optional(), +}) +// Availability and bookmark time describe this account's list item, not the global video. +export const MembershipSchema = z.strictObject({ + availability: z.enum(['available', 'unavailable', 'unknown']), + page_number: z.number().int().positive().optional(), + position_in_page: z.number().int().nonnegative().optional(), + favored_at: instant.optional(), + observed_at: instant, +}) +export const SourceConfigSchema = z.strictObject({ + provider: z.uuid(), + account_mid: NativeIdSchema, + folder_ids: z + .array(NativeIdSchema) + .min(1) + .max(128) + .refine((ids) => new Set(ids).size === ids.length), +}) +export const CollectConfigSchema = z.strictObject({ + max_pages: z.number().int().min(1).max(100).default(10), +}) +export type User = z.infer +export type Folder = z.infer +export type Video = z.infer +export type Membership = z.infer +export type BilibiliSourceConfig = z.infer +export type CollectConfig = z.infer + +export function nativeId(value: unknown): string { + if (typeof value === 'number') { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error('来源身份不是安全整数。') + return String(value) + } + return NativeIdSchema.parse(value) +} + +export function videoUrl(video: Video): string { + return `https://www.bilibili.com/video/${video.bvid ?? `av${video.aid}`}` +} + +export function safeWebUrl(value: string | undefined): string | null { + if (!value) return null + try { + const url = new URL(value) + return ['https:', 'http:'].includes(url.protocol) && !url.username && !url.password + ? url.href + : null + } catch { + return null + } +} + +export type BilibiliRecord = + | { kind: 'user'; record: User } + | { kind: 'folder'; record: Folder } + | { kind: 'video'; record: Video } + | { kind: 'membership'; record: Membership } + +export function recordTitle(value: BilibiliRecord): string { + switch (value.kind) { + case 'user': + return value.record.name || `UID ${value.record.mid}` + case 'folder': + return value.record.title || `收藏夹 ${value.record.id}` + case 'video': + return value.record.title || value.record.bvid || `av${value.record.aid}` + case 'membership': + return value.record.availability === 'unavailable' ? '已失效的收藏条目' : '收藏条目' + } +} + +export function parseRecord(resolver: string, content: unknown): BilibiliRecord | null { + switch (resolver) { + case resolverTypes.user: + return { kind: 'user', record: UserSchema.parse(content) } + case resolverTypes.folder: + return { kind: 'folder', record: FolderSchema.parse(content) } + case resolverTypes.video: + return { kind: 'video', record: VideoSchema.parse(content) } + case resolverTypes.membership: + return { kind: 'membership', record: MembershipSchema.parse(content) } + default: + return null + } +} diff --git a/extensions/bilibili/src/site.ts b/extensions/bilibili/src/site.ts new file mode 100644 index 00000000..38f3b911 --- /dev/null +++ b/extensions/bilibili/src/site.ts @@ -0,0 +1,77 @@ +import { BrowserAccess, type BrowserPage } from '@inkcre/core' +import { parseAccount, parseFavoritePage, parseFolders } from './normalize.ts' +import type { BilibiliSourceConfig } from './schema.ts' + +const apiOrigin = 'https://api.bilibili.com' + +async function readJson(page: BrowserPage, path: string, parameters: Record = {}) { + const url = new URL(path, apiOrigin) + url.search = new URLSearchParams(parameters).toString() + return page.evaluate(async (address) => { + // This runs in the selected logged-in tab; credentials never leave the browser profile. + const response = await fetch(address, { credentials: 'include' }) + if (!response.ok) throw new Error(`B 站请求失败（HTTP ${response.status}）。`) + const value: unknown = await response.json() + return value + }, url.href) +} + +export async function readAccount(page: BrowserPage) { + return parseAccount(await readJson(page, '/x/web-interface/nav')) +} + +export async function discoverFavorites(provider: string, signal: AbortSignal) { + return BrowserAccess.withPage({ provider, target: { kind: 'new' }, signal }, async (page) => { + await page.goto('https://www.bilibili.com/', { waitUntil: 'domcontentloaded' }) + signal.throwIfAborted() + const account = await readAccount(page) + const folders = parseFolders( + await readJson(page, '/x/v3/fav/folder/created/list-all', { + up_mid: account.mid, + }), + account.mid + ) + signal.throwIfAborted() + return { account, folders } + }) +} + +export class FavoriteSession { + constructor( + private readonly page: BrowserPage, + readonly config: BilibiliSourceConfig + ) {} + + async open(folder: string) { + await this.page.goto( + `https://space.bilibili.com/${this.config.account_mid}/favlist?fid=${folder}&ftype=create`, + { + waitUntil: 'domcontentloaded', + } + ) + } + + async account() { + const account = await readAccount(this.page) + if (account.mid !== this.config.account_mid) + throw new Error('所选浏览器的 B 站账号已改变，请重新确认采集源。') + return account + } + + async read(folder: string, pageNumber: number) { + return parseFavoritePage( + await readJson(this.page, '/x/v3/fav/resource/list', { + media_id: folder, + pn: String(pageNumber), + ps: '20', + order: 'mtime', + type: '0', + tid: '0', + platform: 'web', + }), + this.config.account_mid, + folder, + pageNumber + ) + } +} diff --git a/extensions/bilibili/src/source.ts b/extensions/bilibili/src/source.ts new file mode 100644 index 00000000..f77bdec0 --- /dev/null +++ b/extensions/bilibili/src/source.ts @@ -0,0 +1,34 @@ +import { BrowserAccess, type SourceImplementation } from '@inkcre/core' +import { CollectConfigSchema, SourceConfigSchema } from './schema.ts' +import { favoriteGraph } from './graph.ts' +import { startProgress } from './progress.ts' +import { commitBatch } from './repository.ts' +import { FavoriteSession } from './site.ts' +import { collectFavorites } from './collect.ts' + +export const bilibiliSource: SourceImplementation = { + collectConfig: CollectConfigSchema, + backfillConfig: null, + canCollect(source) { + const config = SourceConfigSchema.safeParse(source.config) + return config.success && BrowserAccess.hasBinding(config.data.provider) + }, + async collect(source, _job, options, signal) { + const config = SourceConfigSchema.parse(source.config) + await BrowserAccess.withPage( + { provider: config.provider, target: { kind: 'new' }, signal }, + async (page) => { + await collectFavorites( + config, + startProgress(source.state, config), + options, + signal, + new FavoriteSession(page, config), + async (account, current, next) => { + await commitBatch(source, favoriteGraph(account, current), next) + } + ) + } + ) + }, +} diff --git a/extensions/bilibili/tests/favorites.test.ts b/extensions/bilibili/tests/favorites.test.ts new file mode 100644 index 00000000..a6b4cd72 --- /dev/null +++ b/extensions/bilibili/tests/favorites.test.ts @@ -0,0 +1,366 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + parseAccount, + parseFavoritePage, + parseFolders, + type FavoritePage, +} from '../src/normalize.ts' +import { favoriteGraph } from '../src/graph.ts' +import { SourceConfigSchema, nativeId } from '../src/schema.ts' +import { startProgress, type Progress } from '../src/progress.ts' +import { collectFavorites, type FavoriteSessionPort } from '../src/collect.ts' + +// Invented identities and text; only response field shapes come from observed browser responses. +const account = { mid: '1001', name: '测试账号' } +const info = { + id: 900123, + fid: 9001, + mid: 1001, + upper: account, + title: '技术收藏', + attr: 1, + media_count: 41, +} +const config = SourceConfigSchema.parse({ + provider: 'd089e2c4-e3ec-42a6-953b-42530a726ac2', + account_mid: account.mid, + folder_ids: ['900123'], +}) +const media = (id: number, extra: object = {}) => ({ + id, + type: 2, + bvid: 'BV1234567890', + title: `视频 ${id}`, + intro: '已知简介', + attr: 0, + upper: { mid: 2002, name: '测试 UP 主' }, + fav_time: 1700000000, + ...extra, +}) +function page(ids: number[], more: boolean, number = 1, extra: object = {}): FavoritePage { + return parseFavoritePage( + { code: 0, data: { info, medias: ids.map((id) => media(id)), has_more: more, ...extra } }, + account.mid, + '900123', + number, + '2026-11-06T00:00:00.000Z' + ) +} + +void test('full media_id is kept distinct from raw fid, visibility is a bit, and login is required', () => { + const folders = parseFolders( + { code: 0, data: { list: [info, { ...info, id: 900124, attr: 23 }] } }, + account.mid + ) + assert.equal(folders[0]!.id, '900123') + assert.equal(folders[0]!.is_private, true) + assert.equal(folders[1]!.is_private, true) + assert.throws(() => parseAccount({ code: 0, data: { isLogin: false, mid: 1001 } }), /登录/) + assert.throws( + () => parseFolders({ code: 0, data: { list: [{ ...info, mid: 777 }] } }, account.mid), + /不属于/ + ) + assert.throws(() => nativeId(Number.MAX_SAFE_INTEGER + 1), /安全整数/) +}) + +void test('short pages follow has_more, unavailable placeholders preserve unknown fields, and types are reported', () => { + const value = parseFavoritePage( + { + code: 0, + data: { + info, + has_more: true, + medias: [ + media(11), + media(12, { title: '已失效视频', attr: 9 }), + media(13, { type: 12 }), + { id: 'bad', type: 2 }, + ], + }, + }, + account.mid, + '900123', + 2, + '2026-11-06T00:00:00.000Z' + ) + assert.equal(value.hasMore, true) + assert.equal(value.items.length, 2) + assert.equal(value.unsupported, 1) + assert.equal(value.unresolved, 1) + assert.equal(value.items[1]!.video.title, undefined) + assert.equal(value.items[1]!.video.description, undefined) + assert.equal(value.items[1]!.membership.availability, 'unavailable') + assert.equal(value.items[1]!.membership.page_number, 2) + assert.equal(value.items[1]!.membership.position_in_page, 1) + const record = favoriteGraph(account, value).records.find((entry) => entry.key === 'video:12')! + assert.equal(Object.hasOwn(record.content, 'title'), false) + assert.throws(() => page([], true), /分页为空/) + assert.throws( + () => parseFavoritePage({ code: -101, message: '账号未登录' }, account.mid, '900123', 1), + /未登录/ + ) + assert.throws( + () => parseFavoritePage({ code: 0, data: { info, medias: [] } }, account.mid, '900123', 1), + /结束标记/ + ) +}) + +void test('a shared video has one identity and separate stable membership records in each folder', () => { + const sharedUploader = favoriteGraph(account, page([11, 12], false)) + assert.equal(sharedUploader.records.length, 7) + assert.equal( + new Set(sharedUploader.records.map((entry) => entry.key)).size, + sharedUploader.records.length + ) + const first = favoriteGraph(account, page([11], false)) + const secondPage = { ...page([11], false), folder: { ...page([11], false).folder, id: '900124' } } + const second = favoriteGraph(account, secondPage) + assert.equal( + first.records.find((entry) => entry.resolver.endsWith('.video.v1'))!.key, + second.records.find((entry) => entry.resolver.endsWith('.video.v1'))!.key + ) + assert.notEqual( + first.records.find((entry) => entry.resolver.endsWith('.membership.v1'))!.key, + second.records.find((entry) => entry.resolver.endsWith('.membership.v1'))!.key + ) + assert.deepEqual( + first.relations.map((entry) => entry.content), + ['owns', 'contains', 'item', 'author'] + ) +}) + +void test('optional malformed uploader and time do not discard an identified video', () => { + const value = parseFavoritePage( + { + code: 0, + data: { + info, + has_more: false, + medias: [media(11, { upper: { mid: 0 }, pubtime: Number.MAX_SAFE_INTEGER })], + }, + }, + account.mid, + '900123', + 1 + ) + assert.equal(value.items[0]!.video.aid, '11') + assert.equal(value.items[0]!.video.published_at, undefined) + assert.equal(value.items[0]!.uploader, null) + assert.equal(value.unresolved, 0) +}) + +function session(pages: FavoritePage[], requested: number[]): FavoriteSessionPort { + return { + open: async () => {}, + account: async () => account, + read: async (_folder, number) => { + requested.push(number) + return pages[number - 1]! + }, + } +} + +void test('budgeted runs continue beyond initial pages only after checking head and committed boundary', async () => { + const pages = [page([11, 12], true), page([13], true, 2), page([14], false, 3)] + const requested: number[] = [] + let persisted: Record = {} + const commit = async (_account: unknown, _page: unknown, progress: Progress) => { + persisted = { bilibili: progress } + } + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 2 }, + new AbortController().signal, + session(pages, requested), + commit + ) + assert.equal((persisted.bilibili as Progress).last_run.status, 'partial') + assert.equal((persisted.bilibili as Progress).folders['900123']!.next_page, 3) + requested.length = 0 + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 2 }, + new AbortController().signal, + session(pages, requested), + commit + ) + assert.deepEqual(requested, [1, 2, 3, 1]) + assert.equal((persisted.bilibili as Progress).last_run.status, 'complete') +}) + +void test('a changed head restarts scanning and failed persistence never advances acknowledged progress', async () => { + const pages = [page([11], true), page([12], false, 2)] + let persisted: Record = {} + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 1 }, + new AbortController().signal, + session(pages, []), + async (_account, _page, progress) => { + persisted = { bilibili: progress } + } + ) + const before = structuredClone(persisted) + const requested: number[] = [] + const changed = [page([99], true), page([11], false, 2)] + await assert.rejects( + collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 1 }, + new AbortController().signal, + session(changed, requested), + async () => { + throw new Error('transaction rejected') + } + ), + /transaction rejected/ + ) + assert.deepEqual(requested, [1]) + assert.deepEqual(persisted, before) + const imported: string[] = [] + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 1 }, + new AbortController().signal, + session(changed, []), + async (_account, current, progress) => { + imported.push(...current.ids) + persisted = { bilibili: progress } + } + ) + assert.deepEqual(imported, ['2:99']) +}) + +void test('cancellation and repeated pagination fail without a false completed checkpoint', async () => { + const controller = new AbortController() + controller.abort() + await assert.rejects( + collectFavorites( + config, + startProgress({}, config), + { max_pages: 2 }, + controller.signal, + session([], []), + async () => assert.fail('must not write') + ) + ) + let persisted: Record = {} + await assert.rejects( + collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 2 }, + new AbortController().signal, + session([page([11], true), page([11], true, 2)], []), + async (_account, _page, progress) => { + persisted = { bilibili: progress } + } + ), + /分页没有前进/ + ) + assert.equal((persisted.bilibili as Progress).folders['900123']!.next_page, 2) + assert.notEqual((persisted.bilibili as Progress).last_run.status, 'complete') +}) + +void test('a changed committed boundary restarts even when the first page has not changed', async () => { + const pages = [page([11], true), page([12], true, 2), page([13], false, 3)] + let persisted: Record = {} + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 2 }, + new AbortController().signal, + session(pages, []), + async (_account, _page, progress) => { + persisted = { bilibili: progress } + } + ) + const requested: number[] = [] + const changed = [pages[0]!, page([99], true, 2), pages[2]!] + const imported: string[] = [] + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 1 }, + new AbortController().signal, + session(changed, requested), + async (_account, current, progress) => { + imported.push(...current.ids) + persisted = { bilibili: progress } + } + ) + assert.deepEqual(requested, [1, 2]) + assert.deepEqual(imported, ['2:11']) + assert.equal((persisted.bilibili as Progress).folders['900123']!.next_page, 2) +}) + +void test('an abort during page reading prevents dispatching an uncommitted batch', async () => { + const controller = new AbortController() + const target = session([page([11], false)], []) + target.read = async () => { + controller.abort() + return page([11], false) + } + await assert.rejects( + collectFavorites( + config, + startProgress({}, config), + { max_pages: 1 }, + controller.signal, + target, + async () => assert.fail('must not write an aborted batch') + ) + ) +}) + +void test('a head reorder during the same run prevents a false completed scan', async () => { + let persisted: Record = {} + let headReads = 0 + const target: FavoriteSessionPort = { + open: async () => {}, + account: async () => account, + read: async (_folder, number) => + number === 1 ? page([++headReads === 1 ? 11 : 99], true) : page([12], false, 2), + } + await assert.rejects( + collectFavorites( + config, + startProgress({}, config), + { max_pages: 2 }, + new AbortController().signal, + target, + async (_account, _page, next) => { + persisted = { bilibili: next } + } + ), + /采集期间发生变化/ + ) + const cursor = (persisted.bilibili as Progress).folders['900123']! + assert.equal(cursor.next_page, 2) + assert.equal(cursor.complete, false) + assert.equal(cursor.rows, 1) +}) + +void test('reported count differences from hidden items do not turn a fully traversed scan into an endless restart', async () => { + const target = session([page([11], true), page([12], false, 2)], []) + let result: Progress | undefined + await collectFavorites( + config, + startProgress({}, config), + { max_pages: 2 }, + new AbortController().signal, + target, + async (_account, _page, next) => { + result = next + } + ) + assert.equal(result!.last_run.status, 'complete') + assert.equal(result!.folders['900123']!.reported_count, 41) + assert.equal(result!.folders['900123']!.rows, 2) +}) diff --git a/extensions/bilibili/tsconfig.json b/extensions/bilibili/tsconfig.json new file mode 100644 index 00000000..55268f80 --- /dev/null +++ b/extensions/bilibili/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": ["@vue/tsconfig/tsconfig.dom.json", "../../tsconfig.browser.json"], + "compilerOptions": { + "types": ["vite/client"], + "outDir": "./dist/types", + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo", + "jsx": "preserve", + "jsxImportSource": "vue", + "skipLibCheck": true, + "paths": { + "@/*": ["../../apps/client-web/src/*"], + "@inkcre/core": ["../../packages/core/src/index.ts"], + "@inkcre/core/*": ["../../packages/core/src/*"] + }, + "allowImportingTsExtensions": true, + "strict": true + }, + "include": ["src/**/*", "src/**/*.vue"] +} diff --git a/extensions/bilibili/vite.config.ts b/extensions/bilibili/vite.config.ts new file mode 100644 index 00000000..55d52a4e --- /dev/null +++ b/extensions/bilibili/vite.config.ts @@ -0,0 +1,73 @@ +import { fileURLToPath } from 'node:url' +import path from 'node:path' +import { defineConfig, searchForWorkspaceRoot } from 'vite' +import vue from '@vitejs/plugin-vue' +import { federation } from '@module-federation/vite' + +import mfShared from '../mf-shared' +import extensionPackage from './package.json' +import { + createUiSourceAliases, + isPathInside, + resolveUiSourceForVite, + uiSourceDedupe, +} from '../../scripts/ui-source.mjs' + +const extensionRoot = fileURLToPath(new URL('.', import.meta.url)) +const extensionComponents = fileURLToPath(new URL('./src/components', import.meta.url)) + +export const bilibiliArtifactBase = './' +export const bilibiliBuildTarget = 'es2022' +export const bilibiliBuildOptions = { + target: bilibiliBuildTarget, + outDir: 'dist/client-web', + sourcemap: true, +} +export const bilibiliFederationOptions = { + name: 'extension.bilibili', + filename: 'remoteEntry.js', + manifest: true, + exposes: { '.': path.resolve(__dirname, './src/index.ts') }, + shared: mfShared( + extensionPackage.inkcre.module_federation.host_sdk_version, + extensionPackage.devDependencies['@inkcre/ui-web'] + ), +} + +export default defineConfig(async ({ command }) => { + const uiSource = await resolveUiSourceForVite(command) + const uiSourceComponents = uiSource ? path.resolve(uiSource.root, 'src/components') : null + + return { + plugins: [vue(), federation(bilibiliFederationOptions)], + resolve: { + alias: uiSource ? createUiSourceAliases(uiSource) : [], + ...(uiSource ? { dedupe: uiSourceDedupe } : {}), + }, + ...(uiSource + ? { + optimizeDeps: { exclude: ['@inkcre/ui-web'] }, + server: { + fs: { allow: [searchForWorkspaceRoot(extensionRoot), uiSource.root] }, + }, + } + : {}), + base: bilibiliArtifactBase, + build: bilibiliBuildOptions, + css: { + preprocessorOptions: { + scss: { + additionalData: (source: string, file: string) => { + if ( + isPathInside(extensionComponents, file) || + (uiSourceComponents && isPathInside(uiSourceComponents, file)) + ) { + return `@use "@inkcre/ui-web/styles/mixins" as *;@use "@inkcre/ui-web/styles/functions" as *;${source}` + } + return source + }, + }, + }, + }, + } +}) diff --git a/packages/core/package.json b/packages/core/package.json index 5d6b6e1d..aeab1dd8 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -9,6 +9,10 @@ "module": "./dist/index.js", "types": "./dist/index.d.ts", "exports": { + "./browser-automation/protocol": { + "types": "./dist/browser-automation-protocol.d.ts", + "import": "./dist/browser-automation-protocol.js" + }, ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" @@ -37,6 +41,7 @@ "dayjs": "^1.11.23", "htmlparser2": "10.1.0", "jose": "^5.0.0", + "puppeteer-core": "25.16.0", "semver": "7.8.5", "zod": "^4.6.7", "zod-class": "^0.0.18", diff --git a/packages/core/src/browser-automation/access.ts b/packages/core/src/browser-automation/access.ts new file mode 100644 index 00000000..ec15b2d9 --- /dev/null +++ b/packages/core/src/browser-automation/access.ts @@ -0,0 +1,296 @@ +import type { + Browser, + ConnectionTransport, + Page, +} from 'puppeteer-core/lib/puppeteer/puppeteer-core-browser.js' +import { configStore } from '../config' +import type { PeerRef } from '../peer/peer' +import { + BROWSER_AUTOMATION_PROTOCOL, + BrowserBridgeEventSchema, + BrowserBridgeResponseSchema, + BrowserProviderBindingSchema, + BrowserTabRefSchema, + type BrowserBridgeEvent, + type BrowserBridgeRequest, + type BrowserProviderBinding, + type BrowserTabRef, +} from './protocol' + +export type BrowserPage = Page +export type BrowserPageOptions = { + provider: PeerRef + target: { kind: 'new' } | { kind: 'existing'; tab: BrowserTabRef } + signal?: AbortSignal +} + +export class BrowserAutomationError extends Error { + override name = 'BrowserAutomationError' +} + +const requests = new Map< + string, + { binding: BrowserProviderBinding; resolve(value: unknown): void; reject(error: Error): void } +>() +const sessions = new Map() +const discoveries = new Set<(binding: BrowserProviderBinding) => void>() +let client: string | undefined + +function clientId(): string { + if (client) return client + if (typeof window === 'undefined') + throw new BrowserAutomationError('浏览器能力需要 Web 页面运行时。') + client = crypto.randomUUID() + window.addEventListener('message', receive) + return client +} + +function receive(event: MessageEvent): void { + if (event.source !== window || event.origin !== window.location.origin) return + const response = BrowserBridgeResponseSchema.safeParse(event.data) + if (response.success && response.data.client === client) { + const reply = response.data + const pending = requests.get(reply.request) + if (pending) { + const exactProvider = + pending.binding.provider === reply.provider && pending.binding.instance === reply.instance + if (!exactProvider) return + if (reply.ok) pending.resolve(reply.value) + else pending.reject(new BrowserAutomationError(reply.error ?? '浏览器操作失败。')) + } else if (reply.ok) { + const binding = BrowserProviderBindingSchema.safeParse(reply.value) + if (binding.success) for (const observe of discoveries) observe(binding.data) + } + return + } + const incoming = BrowserBridgeEventSchema.safeParse(event.data) + if (!incoming.success || incoming.data.client !== client) return + sessions.get(incoming.data.session)?.receive(incoming.data) +} + +function bindingKey(): string { + const { INKCRE_PEER_ID, INKCRE_PGREST_URL } = configStore.metaConfig + if (!INKCRE_PEER_ID || !INKCRE_PGREST_URL) + throw new BrowserAutomationError('请先连接信息库，再配置本 Peer 的浏览器绑定。') + return `inkcre.browser-bindings.v1:${INKCRE_PGREST_URL}:${INKCRE_PEER_ID}` +} + +function bindingFor(provider: PeerRef): BrowserProviderBinding { + const binding = BrowserAccess.bindings().find((item) => item.provider === provider) + if (!binding) throw new BrowserAutomationError('本 Web Peer 尚未绑定所选浏览器。') + return binding +} + +function request( + binding: BrowserProviderBinding, + operation: BrowserBridgeRequest['operation'], + parameters: Record = {}, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const id = crypto.randomUUID() + const pageClient = clientId() + return new Promise((resolve, reject) => { + const finish = (error?: Error, value?: unknown) => { + clearTimeout(timeout) + signal?.removeEventListener('abort', abort) + requests.delete(id) + if (error) reject(error) + else resolve(value) + } + const abort = () => finish(new BrowserAutomationError('浏览器操作已取消。')) + const timeout = setTimeout( + () => + finish( + new BrowserAutomationError('浏览器桥未响应；已发送操作的结果可能未知，请勿自动重放。') + ), + 15_000 + ) + requests.set(id, { + binding, + resolve: (value) => finish(undefined, value), + reject: (error) => finish(error), + }) + signal?.addEventListener('abort', abort, { once: true }) + window.postMessage( + { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'request', + client: pageClient, + request: id, + provider: binding.provider, + instance: binding.instance, + operation, + parameters, + } satisfies BrowserBridgeRequest, + window.location.origin + ) + }) +} + +class RelayTransport implements ConnectionTransport { + onmessage?: (message: string) => void + onclose?: () => void + private closed = false + + constructor( + readonly binding: BrowserProviderBinding, + readonly session: string + ) { + sessions.set(session, this) + } + + send(message: string): void { + if (this.closed) throw new BrowserAutomationError('此浏览器会话已经结束。') + void request(this.binding, 'cdp', { session: this.session, message }).catch(() => this.close()) + } + + receive(event: BrowserBridgeEvent): void { + if ( + this.closed || + event.provider !== this.binding.provider || + event.instance !== this.binding.instance + ) + return + if (event.event === 'closed') this.close() + else if (event.data !== undefined) this.onmessage?.(event.data) + } + + close(): void { + if (this.closed) return + this.closed = true + sessions.delete(this.session) + this.onclose?.() + } +} + +/** Local binding is an execution scope, independent of current browser connectivity or login. */ +export class BrowserAccess { + static bindings(): BrowserProviderBinding[] { + try { + const raw: unknown = JSON.parse(localStorage.getItem(bindingKey()) ?? '[]') + return BrowserProviderBindingSchema.array().parse(raw) + } catch { + return [] + } + } + + static hasBinding(provider: PeerRef): boolean { + return this.bindings().some((item) => item.provider === provider && item.protocol === 1) + } + + static bind(binding: BrowserProviderBinding): void { + const parsed = BrowserProviderBindingSchema.parse(binding) + localStorage.setItem( + bindingKey(), + JSON.stringify([ + ...this.bindings().filter((item) => item.provider !== parsed.provider), + parsed, + ]) + ) + } + + static unbind(provider: PeerRef): void { + localStorage.setItem( + bindingKey(), + JSON.stringify(this.bindings().filter((item) => item.provider !== provider)) + ) + } + + /** Discover only extensions whose settings explicitly authorized this exact Web origin. */ + static async discover(signal?: AbortSignal): Promise { + signal?.throwIfAborted() + const found = new Map() + const pageClient = clientId() + const observe = (binding: BrowserProviderBinding) => found.set(binding.provider, binding) + discoveries.add(observe) + try { + window.postMessage( + { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'request', + client: pageClient, + request: crypto.randomUUID(), + operation: 'discover', + parameters: {}, + } satisfies BrowserBridgeRequest, + window.location.origin + ) + await new Promise((resolve, reject) => { + const abort = () => { + clearTimeout(timer) + reject(signal?.reason) + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', abort) + resolve() + }, 1000) + signal?.addEventListener('abort', abort, { once: true }) + }) + return [...found.values()] + } finally { + discoveries.delete(observe) + } + } + + static async tabs(provider: PeerRef, signal?: AbortSignal): Promise { + return BrowserTabRefSchema.array().parse( + await request(bindingFor(provider), 'tabs', {}, signal) + ) + } + + static async withPage( + options: BrowserPageOptions, + action: (page: Page) => Promise + ): Promise { + options.signal?.throwIfAborted() + const binding = bindingFor(options.provider) + // Opening is not cancellable midway: learn the session identity before releasing it. + const session = crypto.randomUUID() + try { + await request(binding, 'open', { session, target: options.target }) + } catch (error) { + await request(binding, 'close', { session }).catch(() => undefined) + throw error + } + const transport = new RelayTransport(binding, session) + let releasing: Promise | undefined + const release = () => { + releasing ??= request(binding, 'close', { session }) + return releasing + } + const abort = () => { + transport.close() + void release().catch(() => undefined) + } + options.signal?.addEventListener('abort', abort, { once: true }) + let browser: Browser | undefined + try { + options.signal?.throwIfAborted() + // The pinned CDP entry avoids the browser entry's Node channel/BiDi import graph. + const { _connectToCdpBrowser } = + await import('puppeteer-core/internal/cdp/BrowserConnector.js') + browser = await _connectToCdpBrowser( + transport, + '', + { defaultViewport: null, protocolTimeout: 30_000 }, + () => undefined + ) + const [page] = await browser.pages() + if (!page) throw new BrowserAutomationError('浏览器没有返回可控制页面。') + options.signal?.throwIfAborted() + const result = await action(page) + options.signal?.throwIfAborted() + return result + } finally { + options.signal?.removeEventListener('abort', abort) + // Await the callback before cleanup; an abort does not roll back already dispatched effects. + try { + await release() + } finally { + transport.close() + await browser?.disconnect() + } + } + } +} diff --git a/packages/core/src/browser-automation/index.ts b/packages/core/src/browser-automation/index.ts new file mode 100644 index 00000000..d0a5e745 --- /dev/null +++ b/packages/core/src/browser-automation/index.ts @@ -0,0 +1,3 @@ +export { BrowserAccess, BrowserAutomationError } from './access' +export type { BrowserPage, BrowserPageOptions } from './access' +export * from './protocol' diff --git a/packages/core/src/browser-automation/protocol.ts b/packages/core/src/browser-automation/protocol.ts new file mode 100644 index 00000000..9087fb3e --- /dev/null +++ b/packages/core/src/browser-automation/protocol.ts @@ -0,0 +1,61 @@ +import { z } from 'zod' + +export const BROWSER_AUTOMATION_PROTOCOL = 'inkcre.browser.v1' +export const BROWSER_AUTOMATION_PORT = 'inkcre-browser-v1' + +export const BrowserProviderBindingSchema = z.object({ + provider: z.uuid(), + instance: z.uuid(), + name: z.string(), + extensionVersion: z.string(), + protocol: z.literal(1), +}) +export type BrowserProviderBinding = z.infer + +export const BrowserTabRefSchema = z.object({ + provider: z.uuid(), + instance: z.uuid(), + epoch: z.uuid(), + reference: z.uuid(), + tabId: z.number().int().nonnegative(), + title: z.string().optional(), + url: z.string().optional(), +}) +export type BrowserTabRef = z.infer + +export const BrowserBridgeRequestSchema = z.object({ + protocol: z.literal(BROWSER_AUTOMATION_PROTOCOL), + kind: z.literal('request'), + client: z.uuid(), + request: z.uuid(), + provider: z.uuid().optional(), + instance: z.uuid().optional(), + operation: z.enum(['discover', 'tabs', 'open', 'cdp', 'close']), + parameters: z.record(z.string(), z.unknown()).default({}), +}) +export type BrowserBridgeRequest = z.infer + +export const BrowserBridgeResponseSchema = z.object({ + protocol: z.literal(BROWSER_AUTOMATION_PROTOCOL), + kind: z.literal('response'), + client: z.uuid(), + request: z.uuid(), + provider: z.uuid(), + instance: z.uuid(), + ok: z.boolean(), + value: z.unknown().optional(), + error: z.string().optional(), +}) +export type BrowserBridgeResponse = z.infer + +export const BrowserBridgeEventSchema = z.object({ + protocol: z.literal(BROWSER_AUTOMATION_PROTOCOL), + kind: z.literal('event'), + client: z.uuid(), + provider: z.uuid(), + instance: z.uuid(), + session: z.uuid(), + event: z.enum(['message', 'closed']), + data: z.string().optional(), +}) +export type BrowserBridgeEvent = z.infer diff --git a/packages/core/src/database/database.generated.ts b/packages/core/src/database/database.generated.ts index 1273484e..45c386dd 100644 --- a/packages/core/src/database/database.generated.ts +++ b/packages/core/src/database/database.generated.ts @@ -253,6 +253,32 @@ export type Database = { }, ] } + collection_records: { + Row: { + block: number + namespace: string + native_key: string + } + Insert: { + block: number + namespace: string + native_key: string + } + Update: { + block?: number + namespace?: string + native_key?: string + } + Relationships: [ + { + foreignKeyName: 'collection_records_block_fkey' + columns: ['block'] + isOneToOne: false + referencedRelation: 'blocks' + referencedColumns: ['id'] + }, + ] + } configs: { Row: { created_at: string @@ -855,6 +881,19 @@ export type Database = { [_ in never]: never } Functions: { + commit_source_batch: { + Args: { + expected_config: Json + expected_state: Json + namespace: string + next_state: Json + records: Json + relations: Json + root_key?: string + source_id: number + } + Returns: Json + } read_storage_blob: { Args: { blob_id: string }; Returns: unknown } renew_peer_lease: { Args: { peer: string; ttl_seconds: number } diff --git a/packages/core/src/database/runtime-contract.generated.json b/packages/core/src/database/runtime-contract.generated.json index d9a1bc19..ac2f0bd0 100644 --- a/packages/core/src/database/runtime-contract.generated.json +++ b/packages/core/src/database/runtime-contract.generated.json @@ -1,6 +1,6 @@ { "format": 1, - "revision": "peer-jwt-signing-defaults-v1", + "revision": "source-batch-persistence-v1", "protocol": { "format": 1, "schema": "inkcre" diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index bcfefaaf..adb5f08f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -10,6 +10,7 @@ // Core singleton export { store } from './store' +export * from './browser-automation' // Base layer export { APIError, DBAPIClient, rawPostgrestFetch } from './base' @@ -117,6 +118,8 @@ export { SourceForm, SourceType, SourceManager, + SourceResolver, + SourceAnchorSchema, SOURCE_COLLECT_JOB_TYPE, SOURCE_BACKFILL_JOB_TYPE, type SourceImplementation, diff --git a/packages/core/src/info-base/resolvers/contracts.ts b/packages/core/src/info-base/resolvers/contracts.ts index 150d36f3..26effdb2 100644 --- a/packages/core/src/info-base/resolvers/contracts.ts +++ b/packages/core/src/info-base/resolvers/contracts.ts @@ -9,6 +9,7 @@ export const CORE_RESOLVER_IDS = [ 'core.epub.v1', 'core.zip.v1', 'core.file.v1', + 'core.source.v1', ] as const export type CoreResolverId = (typeof CORE_RESOLVER_IDS)[number] diff --git a/packages/core/src/info-base/resolvers/index.ts b/packages/core/src/info-base/resolvers/index.ts index 66cafa38..9c93e373 100644 --- a/packages/core/src/info-base/resolvers/index.ts +++ b/packages/core/src/info-base/resolvers/index.ts @@ -47,6 +47,7 @@ import { TextResolver } from './text' import { MarkdownResolver } from './markdown' import { VideoResolver } from './video' import { ZipResolver } from './zip' +import { SourceResolver } from '../../source/resolver' export function registerCoreResolvers(): void { for (const resolverClass of [ @@ -60,6 +61,7 @@ export function registerCoreResolvers(): void { EpubResolver, ZipResolver, FileResolver, + SourceResolver, ]) { Resolver.register(resolverClass.type, resolverClass) } diff --git a/packages/core/src/job/manager.ts b/packages/core/src/job/manager.ts index e6fe5af9..d86ce9b0 100644 --- a/packages/core/src/job/manager.ts +++ b/packages/core/src/job/manager.ts @@ -118,6 +118,12 @@ export class JobManager { private static async runAdmitted(id: JobRef): Promise { const candidate = await Job.get(id) if (!this.accepting || candidate.status !== JobStatus.PENDING) return false + if (candidate.abort_requested) { + const claimed = await this.claim(candidate) + if (!claimed) return false + await this.observeExecution(claimed, () => this.close(claimed, JobStatus.ABORTED)) + return true + } let prepared: [JobHandler, unknown] | null try { prepared = await this.prepare(candidate) @@ -126,7 +132,7 @@ export class JobManager { if (!this.accepting) return false const claimed = await this.claim(candidate) if (!claimed) return false - if (!this.accepting) { + if (!this.accepting || claimed.abort_requested) { await this.observeExecution(claimed, () => this.close(claimed, JobStatus.ABORTED)) return true } @@ -140,7 +146,7 @@ export class JobManager { if (!claimed) return false // stopWorker may have run while claim was in flight. Do not start a handler // after its Extension resources have been disposed. - if (!this.accepting) { + if (!this.accepting || claimed.abort_requested) { await this.observeExecution(claimed, () => this.close(claimed, JobStatus.ABORTED)) return true } diff --git a/packages/core/src/source/index.ts b/packages/core/src/source/index.ts index 1e77ecd0..7f0f5e7d 100644 --- a/packages/core/src/source/index.ts +++ b/packages/core/src/source/index.ts @@ -1,3 +1,4 @@ export * from './source-type' export * from './source' export * from './manager' +export * from './resolver' diff --git a/packages/core/src/source/manager.ts b/packages/core/src/source/manager.ts index de99ffbd..e2f0cc64 100644 --- a/packages/core/src/source/manager.ts +++ b/packages/core/src/source/manager.ts @@ -16,6 +16,8 @@ export interface SourceImplementation< > { collectConfig: CollectConfig backfillConfig: BackfillConfig + canCollect?(source: Source, config: z.output): boolean | Promise + canBackfill?(source: Source, config: unknown): boolean | Promise collect( source: Source, job: Job, @@ -42,7 +44,8 @@ export class SourceManager { this.implementations.set(type, implementation) } - static unregister(type: string): void { + static unregister(type: string, implementation?: SourceImplementation): void { + if (implementation && this.implementations.get(type) !== implementation) return this.implementations.delete(type) } @@ -60,7 +63,13 @@ export class SourceManager { JobManager.registerHandler(SOURCE_COLLECT_JOB_TYPE, { parameters: SourceCommandParameters, async canHandle(parameters) { - return (await SourceManager.resolve(parameters.source)) !== null + const resolved = await SourceManager.resolve(parameters.source) + if (!resolved) return false + const [source, implementation] = resolved + const config = implementation.collectConfig.safeParse(parameters.config) + // Invalid commands must be claimed and fail in handle, rather than remain pending forever. + if (!config.success) return true + return implementation.canCollect ? implementation.canCollect(source, config.data) : true }, async handle(job, parameters, signal) { const resolved = await SourceManager.resolve(parameters.source) @@ -75,7 +84,12 @@ JobManager.registerHandler(SOURCE_BACKFILL_JOB_TYPE, { parameters: SourceCommandParameters, async canHandle(parameters) { const resolved = await SourceManager.resolve(parameters.source) - return resolved !== null && resolved[1].backfillConfig !== null && !!resolved[1].backfill + if (!resolved) return false + const [source, implementation] = resolved + if (!implementation.backfillConfig || !implementation.backfill) return false + const config = implementation.backfillConfig.safeParse(parameters.config) + if (!config.success) return true + return implementation.canBackfill ? implementation.canBackfill(source, config.data) : true }, async handle(job, parameters, signal) { const resolved = await SourceManager.resolve(parameters.source) diff --git a/packages/core/src/source/resolver.ts b/packages/core/src/source/resolver.ts new file mode 100644 index 00000000..8e5b7dd1 --- /dev/null +++ b/packages/core/src/source/resolver.ts @@ -0,0 +1,38 @@ +import { z } from 'zod' +import { Resolver } from '../info-base/resolvers/base' +import { ResolverContentError, type ProjectionOptions } from '../info-base/resolvers/contracts' +import { decodeUnicode } from '../info-base/resolvers/decode' + +export const SourceAnchorSchema = z.strictObject({ + id: z.number().int(), + type: z.string(), + nickname: z.string().nullable().default(null), +}) + +/** The Source anchor describes collection provenance; its outgoing relations lead to business data. */ +export class SourceResolver extends Resolver { + static readonly type = 'core.source.v1' + + protected async _getSolvedContent(options: ProjectionOptions): Promise { + const raw = decodeUnicode(await this.getRawContent(options), SourceResolver.type) + let decoded: unknown + try { + decoded = JSON.parse(raw) + } catch { + throw new ResolverContentError(SourceResolver.type, 'expected a JSON Source anchor') + } + const parsed = SourceAnchorSchema.safeParse(decoded) + if (!parsed.success) + throw new ResolverContentError( + SourceResolver.type, + 'expected Source id, type and optional nickname' + ) + const source = parsed.data + const name = source.nickname ? ` named ${source.nickname}` : '' + return `Source ${source.id}${name} uses ${source.type}.` + } + + async getText(options: ProjectionOptions = {}): Promise { + return this.getSolvedContent(options) + } +} diff --git a/packages/core/tsdown.config.ts b/packages/core/tsdown.config.ts index 1ca641c0..a632136a 100644 --- a/packages/core/tsdown.config.ts +++ b/packages/core/tsdown.config.ts @@ -12,6 +12,14 @@ const common = { // Separate graphs keep the SDK's declarations independent of Vue peer dependencies. export default defineConfig([ - { ...common, entry: ['src/index.ts'], outDir: 'dist', clean: false }, + { + ...common, + entry: { + index: 'src/index.ts', + 'browser-automation-protocol': 'src/browser-automation/protocol.ts', + }, + outDir: 'dist', + clean: false, + }, { ...common, entry: ['src/sdk.ts'], outDir: 'dist/sdk', clean: false }, ]) diff --git a/patches/puppeteer-core@25.16.0.patch b/patches/puppeteer-core@25.16.0.patch new file mode 100644 index 00000000..b6f1a2ce --- /dev/null +++ b/patches/puppeteer-core@25.16.0.patch @@ -0,0 +1,51 @@ +diff --git a/src/cdp/ExtensionTransport.ts b/src/cdp/ExtensionTransport.ts +--- a/src/cdp/ExtensionTransport.ts ++++ b/src/cdp/ExtensionTransport.ts +@@ -190,8 +190,8 @@ + }); + } + +- close(): void { ++ close(): Promise { + chrome.debugger.onEvent.removeListener(this.#debuggerEventHandler); +- void chrome.debugger.detach({tabId: this.#tabId}); ++ return chrome.debugger.detach({tabId: this.#tabId}); + } + } +diff --git a/lib/puppeteer/cdp/ExtensionTransport.js b/lib/puppeteer/cdp/ExtensionTransport.js +--- a/lib/puppeteer/cdp/ExtensionTransport.js ++++ b/lib/puppeteer/cdp/ExtensionTransport.js +@@ -169,7 +169,7 @@ + } + close() { + chrome.debugger.onEvent.removeListener(this.#debuggerEventHandler); +- void chrome.debugger.detach({ tabId: this.#tabId }); ++ return chrome.debugger.detach({ tabId: this.#tabId }); + } + } + //# sourceMappingURL=ExtensionTransport.js.map +\ No newline at end of file +diff --git a/lib/puppeteer/cdp/ExtensionTransport.d.ts b/lib/puppeteer/cdp/ExtensionTransport.d.ts +--- a/lib/puppeteer/cdp/ExtensionTransport.d.ts ++++ b/lib/puppeteer/cdp/ExtensionTransport.d.ts +@@ -23,6 +23,6 @@ + */ + constructor(tabId: number); + send(message: string): void; +- close(): void; ++ close(): Promise; + } + //# sourceMappingURL=ExtensionTransport.d.ts.map +\ No newline at end of file +diff --git a/lib/types.d.ts b/lib/types.d.ts +--- a/lib/types.d.ts ++++ b/lib/types.d.ts +@@ -3102,7 +3102,7 @@ + onmessage?: (message: string) => void; + onclose?: () => void; + send(message: string): void; +- close(): void; ++ close(): Promise; + } + + /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09ad34fb..d3d463a8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,7 @@ overrides: patchedDependencies: '@inkcre/ui-web@2.5.0': 2b0f967ae08c3559202523c4ccb3ed4769ff9d8863a52c600259231171d56051 + puppeteer-core@25.16.0: ca9d8d4e7430e6c3ff5cf5f5802f5aa087cc157c8a5d3c9547a27426b22d8316 importers: @@ -227,6 +228,9 @@ importers: '@mozilla/readability': specifier: ^0.6.0 version: 0.6.0 + puppeteer-core: + specifier: 25.16.0 + version: 25.16.0(patch_hash=ca9d8d4e7430e6c3ff5cf5f5802f5aa087cc157c8a5d3c9547a27426b22d8316) vue: specifier: ^3.5.44 version: 3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2) @@ -253,6 +257,55 @@ importers: specifier: ^0.20.6 version: 0.20.27(@rspack/core@2.1.5)(@types/node@22.17.2)(eslint@9.39.5(jiti@2.7.0))(jiti@2.7.0)(lightningcss@1.33.0)(oxc-parser@0.131.0)(rolldown@1.2.0)(rollup@4.59.0)(sass-embedded@1.90.0)(sass@1.101.6)(yaml@2.9.0) + extensions/bilibili: + dependencies: + '@inkcre/core': + specifier: workspace:* + version: link:../../packages/core + '@vueuse/core': + specifier: ^14.0.0 + version: 14.4.0(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + pinia: + specifier: ^3.0.0 + version: 3.0.4(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + vue: + specifier: ^3.5.25 + version: 3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2) + vue-router: + specifier: ^4.5.0 + version: 4.6.4(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + devDependencies: + '@inkcre/ext-dev-utils': + specifier: workspace:* + version: link:../../packages/ext-dev-utils + '@inkcre/ui-web': + specifier: https://github.com/InKCre/ui/releases/download/%40inkcre%2Fui-web%402.7.0/inkcre-ui-web-2.7.0.tgz + version: https://github.com/InKCre/ui/releases/download/%40inkcre%2Fui-web%402.7.0/inkcre-ui-web-2.7.0.tgz(unocss@66.7.5(vite@7.3.6(@types/node@22.17.2)(jiti@2.7.0)(lightningcss@1.33.0)(sass-embedded@1.90.0)(sass@1.101.6)(yaml@2.9.0)))(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + '@module-federation/vite': + specifier: ^1.1.0 + version: 1.7.1(rollup@4.59.0) + '@vitejs/plugin-vue': + specifier: ^6.0.1 + version: 6.0.3(vite@7.3.6(@types/node@22.17.2)(jiti@2.7.0)(lightningcss@1.33.0)(sass-embedded@1.90.0)(sass@1.101.6)(yaml@2.9.0))(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + '@vue/tsconfig': + specifier: 'catalog:' + version: 0.9.1(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + sass-embedded: + specifier: ^1.90.0 + version: 1.90.0 + typescript: + specifier: npm:typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2 + version: typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2 + vite: + specifier: ^7.0.6 + version: 7.3.6(@types/node@22.17.2)(jiti@2.7.0)(lightningcss@1.33.0)(sass-embedded@1.90.0)(sass@1.101.6)(yaml@2.9.0) + vue-tsc: + specifier: 'catalog:' + version: 3.3.8(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2) + zod: + specifier: ^4.6.7 + version: 4.6.7 + extensions/caldav: dependencies: '@inkcre/core': @@ -646,6 +699,9 @@ importers: jose: specifier: ^5.0.0 version: 5.10.0 + puppeteer-core: + specifier: 25.16.0 + version: 25.16.0(patch_hash=ca9d8d4e7430e6c3ff5cf5f5802f5aa087cc157c8a5d3c9547a27426b22d8316) semver: specifier: 7.8.5 version: 7.8.5 @@ -2327,6 +2383,19 @@ packages: '@poppinss/exception@1.2.3': resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + '@puppeteer/browsers@3.2.6': + resolution: {integrity: sha512-MJP29gVoHznmqmwhH1tlZqzfeQ3aylTpSSa6WKZXtkwYUH02i9imBmWGTyZrbnAou5paX78SE4jMAUqnEPf8ug==} + engines: {node: '>=22.12.0'} + hasBin: true + peerDependencies: + proxy-agent: '>=8.0.1' + yauzl: ^2.10.0 || ^3.4.0 + peerDependenciesMeta: + proxy-agent: + optional: true + yauzl: + optional: true + '@quansync/fs@1.0.0': resolution: {integrity: sha512-4TJ3DFtlf1L5LDMaM6CanJ/0lckGNtJcMjQ1NAV6zDmA0tEHKZtxNKin8EgPaVX1YzljbxckyT2tJrpQKAtngQ==} @@ -3510,6 +3579,12 @@ packages: engines: {node: '>=12.13.0'} hasBin: true + chromium-bidi@157.0.8090-0: + resolution: {integrity: sha512-KJw7IQgqXwxpJgJlEIPg5mr2ZoqA+jkTRtc5EABI7sUJ3bu3cpI7NuEBoj4Gn1AsF78CaT9So5WQWspK3/wYag==} + engines: {node: '>=20.19.0 <22.0.0 || >=22.12.0'} + peerDependencies: + devtools-protocol: '*' + ci-info@4.4.0: resolution: {integrity: sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==} engines: {node: '>=8'} @@ -3533,6 +3608,10 @@ packages: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} + cliui@9.0.1: + resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} + engines: {node: '>=20'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -3751,6 +3830,9 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + devtools-protocol@0.0.1704951: + resolution: {integrity: sha512-GQhQOs9u/Mvm1zMtJIH872JS4AskuZdUSsPNPk4ChahtWAUqgz6atEtIsMt+nO7MqF1Ic4HxII/k8Cu36JivNg==} + dom-serializer@2.0.0: resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} @@ -4578,6 +4660,10 @@ packages: mlly@1.8.2: resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + modern-tar@0.8.6: + resolution: {integrity: sha512-LYSks0tNVqbtez/9OB0YFP6qLRSqkdAxJWOQpVaR67lNTbj2zNLyYpowkRMia5lXLCZKzxwoxkxH2MbbBR2aEg==} + engines: {node: '>=18.0.0'} + mrmime@2.0.1: resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==} engines: {node: '>=10'} @@ -5054,6 +5140,10 @@ packages: resolution: {integrity: sha512-LjgDO2zPtoXP2wJpDjZrGdojii1uqO0cnwKoIoUzkfS98HDmbeiGmYiXo3lXeFlq2xvne1QFQhwYXSUCLKtEuA==} engines: {node: '>=12.20'} + puppeteer-core@25.16.0: + resolution: {integrity: sha512-VA5Pf54N8shfopNkZZXCdiiwfnnY8ZJYAMqFQRhlH5mGUhfwf53Ar770GPtdpTDLUqvQeYpIBtC9/G2Jij2IJw==} + engines: {node: '>=22.12.0'} + quansync@0.2.11: resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} @@ -5574,6 +5664,9 @@ packages: type-level-regexp@0.1.17: resolution: {integrity: sha512-wTk4DH3cxwk196uGLK/E9pE45aLfeKJacKmcEgEOA/q5dnPGNxXt0cfYdFxb57L+sEpf1oJH4Dnx/pnRcku9jg==} + typed-query-selector@2.12.3: + resolution: {integrity: sha512-/dtuMZyutHLlrxWhhC1uib00ELUr2dZ8C9B8YzZYsJmeE3V56OxRblrSM2UaF142pcMtak7ehZ6+poWGeehNIg==} + typedarray@0.0.6: resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} @@ -5893,6 +5986,9 @@ packages: resolution: {integrity: sha512-rQicL7OwuqWdQWI33JkSXKcp7cuv1mJG8u3jRQwx/8aDsmhbTHs9ZRmNYOL+LX0wX8edIEQX8jj4bB60GoXtKA==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} + webdriver-bidi-protocol@1.0.1: + resolution: {integrity: sha512-WXWs/aXF2a6Al1Z2/V/7Gp2oPZOAAmHjJOTmv4KCXRJ122IMj3tSSMc6a4q3K4QOmhQbse/Ahr+oTVJOU2dUiw==} + webpack-virtual-modules@0.6.2: resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} @@ -5966,6 +6062,18 @@ packages: utf-8-validate: optional: true + ws@8.22.1: + resolution: {integrity: sha512-oyL6OPYme/CmiFvNuGzeoxCJ4nOrZE2O1FwM9MWM1AsGri6nLXB2pg+lNA8aLeXMSav0J6eIVS1j6vUPzUtK9g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + wsl-utils@0.1.0: resolution: {integrity: sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==} engines: {node: '>=18'} @@ -6012,6 +6120,10 @@ packages: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} + yargs-parser@22.0.0: + resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yargs@17.7.2: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} @@ -6020,6 +6132,10 @@ packages: resolution: {integrity: sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==} engines: {node: '>=12'} + yargs@18.2.0: + resolution: {integrity: sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -7596,6 +7712,11 @@ snapshots: '@poppinss/exception@1.2.3': {} + '@puppeteer/browsers@3.2.6': + dependencies: + modern-tar: 0.8.6 + yargs: 18.2.0 + '@quansync/fs@1.0.0': dependencies: quansync: 1.0.0 @@ -8676,6 +8797,12 @@ snapshots: transitivePeerDependencies: - supports-color + chromium-bidi@157.0.8090-0(devtools-protocol@0.0.1704951): + dependencies: + devtools-protocol: 0.0.1704951 + mitt: 3.0.1 + zod: 4.6.7 + ci-info@4.4.0: {} citty@0.2.2: {} @@ -8697,6 +8824,12 @@ snapshots: strip-ansi: 6.0.1 wrap-ansi: 7.0.0 + cliui@9.0.1: + dependencies: + string-width: 7.2.0 + strip-ansi: 7.2.0 + wrap-ansi: 9.0.2 + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -8880,6 +9013,8 @@ snapshots: detect-libc@2.1.2: {} + devtools-protocol@0.0.1704951: {} + dom-serializer@2.0.0: dependencies: domelementtype: 2.3.0 @@ -9676,6 +9811,8 @@ snapshots: pkg-types: 1.3.1 ufo: 1.6.4 + modern-tar@0.8.6: {} + mrmime@2.0.1: {} ms@2.1.3: {} @@ -10064,6 +10201,20 @@ snapshots: dependencies: escape-goat: 4.0.0 + puppeteer-core@25.16.0(patch_hash=ca9d8d4e7430e6c3ff5cf5f5802f5aa087cc157c8a5d3c9547a27426b22d8316): + dependencies: + '@puppeteer/browsers': 3.2.6 + chromium-bidi: 157.0.8090-0(devtools-protocol@0.0.1704951) + devtools-protocol: 0.0.1704951 + typed-query-selector: 2.12.3 + webdriver-bidi-protocol: 1.0.1 + ws: 8.22.1 + transitivePeerDependencies: + - bufferutil + - proxy-agent + - utf-8-validate + - yauzl + quansync@0.2.11: {} quansync@1.0.0: {} @@ -10599,6 +10750,8 @@ snapshots: type-level-regexp@0.1.17: {} + typed-query-selector@2.12.3: {} + typedarray@0.0.6: {} typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2: @@ -10968,6 +11121,8 @@ snapshots: transitivePeerDependencies: - supports-color + webdriver-bidi-protocol@1.0.1: {} + webpack-virtual-modules@0.6.2: {} when-exit@2.1.5: {} @@ -11034,12 +11189,14 @@ snapshots: wrap-ansi@9.0.2: dependencies: - ansi-styles: 6.2.1 + ansi-styles: 6.2.3 string-width: 7.2.0 strip-ansi: 7.2.0 ws@8.21.0: {} + ws@8.22.1: {} + wsl-utils@0.1.0: dependencies: is-wsl: 3.1.0 @@ -11136,6 +11293,8 @@ snapshots: yargs-parser@21.1.1: {} + yargs-parser@22.0.0: {} + yargs@17.7.2: dependencies: cliui: 8.0.1 @@ -11156,6 +11315,15 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + yargs@18.2.0: + dependencies: + cliui: 9.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + string-width: 8.2.2 + y18n: 5.0.8 + yargs-parser: 22.0.0 + yocto-queue@0.1.0: optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5f0ff9cd..d4d46a9e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -57,5 +57,7 @@ minimumReleaseAgeExclude: - marked@18.1.2 patchedDependencies: + # shortcut: 上游 ExtensionTransport.close 尚未返回 detach Promise；上游修复后移除此补丁。 + puppeteer-core@25.16.0: patches/puppeteer-core@25.16.0.patch # shortcut: 修复剩余 2.5.0 消费者重复渲染输入值和 ID；升级到上游修复版时移除对应补丁。 '@inkcre/ui-web@2.5.0': patches/@inkcre__ui-web@2.5.0.patch diff --git a/scripts/check-local-runtime-contract.mjs b/scripts/check-local-runtime-contract.mjs index 022cefff..ac7dbb30 100644 --- a/scripts/check-local-runtime-contract.mjs +++ b/scripts/check-local-runtime-contract.mjs @@ -138,7 +138,7 @@ for (const legacyExport of ['httpAdapter', 'envAdapter', 'createEnvAdapter']) { } } -const localTargets = svcConfig.dev?.targets +const localTargets = svcConfig.dev?.resources ?? svcConfig.dev?.targets if (!localTargets) { errors.push('svc.json must declare development targets') } else { @@ -173,18 +173,17 @@ if (!localTargets) { if (target.scope !== 'worktree') { errors.push(`SVC target "${name}" must be worktree-scoped`) } - if ( - target.probe?.kind !== 'exec' || - JSON.stringify(target.probe.argv) !== JSON.stringify(expected.probe) - ) { + const probe = target.observations?.ready?.probe ?? target.probe + const provision = target.actions?.ensure ?? target.provision + if (probe?.kind !== 'exec' || JSON.stringify(probe.argv) !== JSON.stringify(expected.probe)) { errors.push(`SVC target "${name}" must use the identity probe that proves dev.instance`) } - if (JSON.stringify(target.provision?.argv) !== JSON.stringify(expected.command)) { + if (JSON.stringify(provision?.argv) !== JSON.stringify(expected.command)) { errors.push(`SVC target "${name}" must provision through scripts/dev.mjs`) } if ( expected.environment && - JSON.stringify(target.provision?.env) !== JSON.stringify(expected.environment) + JSON.stringify(provision?.env) !== JSON.stringify(expected.environment) ) { errors.push(`SVC target "${name}" must commit only the portable local provider default`) } @@ -237,7 +236,10 @@ for (const required of [ if (remoteCompose.includes('eval ') || !remoteCompose.includes('"$docker_bin" compose')) { errors.push('remote Compose runner must preserve argv without eval or shell command construction') } -const committedDatabaseEnvironment = svcConfig.dev.targets.database.provision.env +const committedDatabaseEnvironment = + svcConfig.dev.resources?.database?.actions?.ensure?.env ?? + svcConfig.dev.targets?.database?.provision?.env ?? + {} if ( committedDatabaseEnvironment?.INKCRE_DATABASE_SSH_TARGET || committedDatabaseEnvironment?.INKCRE_DATABASE_SSH_DOCKER_BIN || diff --git a/scripts/check-package-contract.mjs b/scripts/check-package-contract.mjs index 55f20aad..84b6af61 100644 --- a/scripts/check-package-contract.mjs +++ b/scripts/check-package-contract.mjs @@ -38,6 +38,7 @@ await Promise.all( 'apps/webext/.output/chrome-mv3/options.html', 'apps/webext/.output/chrome-mv3/popup.html', 'extensions/github/dist/web/remoteEntry.js', + 'extensions/bilibili/dist/web/remoteEntry.js', 'extensions/caldav/dist/web/remoteEntry.js', 'extensions/capture/dist/web/remoteEntry.js', 'extensions/local-files/dist/web/remoteEntry.js', diff --git a/scripts/check-workspace-contract.mjs b/scripts/check-workspace-contract.mjs index 917d94c8..fbb71f28 100644 --- a/scripts/check-workspace-contract.mjs +++ b/scripts/check-workspace-contract.mjs @@ -6,6 +6,12 @@ const repoRoot = fileURLToPath(new URL('..', import.meta.url)) const jsonOutput = process.argv.includes('--json') const workspaces = [ + { + path: 'extensions/bilibili', + name: '@inkcre/ext-bilibili', + requiredScripts: ['build', 'dev', 'type-check'], + builder: 'vite', + }, { path: 'extensions/local-files', name: '@inkcre/ext-local-files', diff --git a/scripts/database-contract-source.mjs b/scripts/database-contract-source.mjs new file mode 100644 index 00000000..2d88a403 --- /dev/null +++ b/scripts/database-contract-source.mjs @@ -0,0 +1,16 @@ +import { projectDatabaseCompatibilityContract } from './core-release-lib.mjs' + +/** Local development may consume an owner runtime without inventing a published release. */ +export function validateDevelopmentContract(descriptor, contract) { + if (descriptor.profile !== 'development' || descriptor.owner_repository !== 'InKCre/impl-py') { + throw new Error('local type generation requires an impl-py development runtime') + } + if (descriptor.converging) throw new Error('database owner is still converging') + if (contract.revision !== descriptor.contract_revision) { + throw new Error('local runtime and exported source contract revisions differ') + } + if (!contract.migration_heads?.includes(descriptor.migration_head)) { + throw new Error('local runtime and exported source migration heads differ') + } + return projectDatabaseCompatibilityContract(contract) +} diff --git a/scripts/database-provider.mjs b/scripts/database-provider.mjs index 1ce0b079..732e848d 100644 --- a/scripts/database-provider.mjs +++ b/scripts/database-provider.mjs @@ -34,7 +34,11 @@ function optionalJson(path) { } function provisionEnvironment(config) { - return config.dev?.targets?.database?.provision?.env ?? {} + return ( + config.dev?.resources?.database?.actions?.ensure?.env ?? + config.dev?.targets?.database?.provision?.env ?? + {} + ) } function declaredProviderEnvironment() { diff --git a/scripts/doctor.mjs b/scripts/doctor.mjs index 984b9d8f..6eb8158f 100644 --- a/scripts/doctor.mjs +++ b/scripts/doctor.mjs @@ -70,7 +70,7 @@ record( 'docs/_shared must be initialized' ) -const localTargets = svcConfig.dev?.targets +const localTargets = svcConfig.dev?.resources ?? svcConfig.dev?.targets record( 'svc-dev-targets', localTargets?.web?.scope === 'worktree' && diff --git a/scripts/prepare-native-checkout.mjs b/scripts/prepare-native-checkout.mjs new file mode 100644 index 00000000..43d81f1f --- /dev/null +++ b/scripts/prepare-native-checkout.mjs @@ -0,0 +1,190 @@ +import { execFile } from 'node:child_process' +import { createHash } from 'node:crypto' +import { lstat, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises' +import { dirname, join, resolve, sep } from 'node:path' +import { pathToFileURL } from 'node:url' +import { promisify } from 'node:util' +import semver from 'semver' + +const digest = (value) => createHash('sha256').update(value).digest('hex') +const executeFile = promisify(execFile) +const execute = async (command, args, cwd) => + ( + await executeFile(command, args, { + cwd, + encoding: 'utf8', + }) + ).stdout + +const listedFiles = ['ls-files', '-z', '--cached', '--others', '--exclude-standard'] +function includeSource(file) { + if (!file || file.startsWith('.runtime/')) return false + const dotenv = /(^|\/)\.env(?:\.|$)/.test(file) + return !dotenv || /(^|\/)\.env(?:\.[^/]*)?\.(example|sample)$/.test(file) +} + +async function packages(root) { + const result = new Map() + for (const family of ['packages', 'apps', 'extensions']) { + for (const entry of await readdir(join(root, family), { withFileTypes: true }).catch( + () => [] + )) { + if (!entry.isDirectory()) continue + const path = join(family, entry.name, 'package.json') + const text = await readFile(join(root, path), 'utf8').catch(() => null) + if (text) { + const metadata = JSON.parse(text) + result.set(metadata.name, { path, metadata }) + } + } + } + return result +} + +async function intents(root) { + return (await readdir(join(root, '.changeset')).catch(() => [])) + .filter((name) => name.endsWith('.md') && name.toLowerCase() !== 'readme.md') + .sort() +} + +async function verifySource(source, files, run) { + const listed = String(await run('git', listedFiles, source)) + .split('\0') + .filter(includeSource) + const currentFiles = [] + for (const file of listed) { + const stat = await lstat(join(source, file)).catch(() => null) + if (stat?.isFile()) currentFiles.push(file) + } + if (JSON.stringify(currentFiles.sort()) !== JSON.stringify(Object.keys(files).sort())) + throw new Error('Source file set changed after snapshot; refresh the candidate') + for (const [file, hash] of Object.entries(files)) { + const current = await readFile(join(source, file)).catch(() => null) + if (!current || digest(current) !== hash) + throw new Error( + `Source changed after snapshot: ${file}; refresh the candidate without reverting source work` + ) + } +} + +/** Prepare validation coordinates with the existing Changesets command; never edit source versions. */ +export async function prepareNativeCheckout({ + sourceDirectory, + outputDirectory, + offline = false, + run = execute, +}) { + const source = await realpath(sourceDirectory) + const output = resolve(outputDirectory) + if (output === source || source.startsWith(output + sep)) + throw new Error('Candidate cannot replace the source or its ancestor') + if ((await readdir(output).catch(() => [])).length) + throw new Error('Candidate output must be empty') + const candidate = join(output, 'workspace') + await mkdir(candidate, { recursive: true }) + const revision = String(await run('git', ['rev-parse', 'HEAD'], source)).trim() + if (!/^[a-f0-9]{40}$/.test(revision)) throw new Error('Source requires one existing Git revision') + const sourceModified = !!String(await run('git', ['status', '--porcelain'], source)).trim() + const paths = String(await run('git', listedFiles, source)) + .split('\0') + .filter(Boolean) + .sort() + const files = {} + for (const file of paths) { + if (!includeSource(file)) continue + const origin = join(source, file) + const stat = await lstat(origin).catch(() => null) + if (!stat || stat.isDirectory()) continue // Git submodule content is not part of git archive either. + if (stat.isSymbolicLink()) throw new Error(`Candidate source contains a symbolic link: ${file}`) + const bytes = await readFile(origin) + const target = join(candidate, file) + await mkdir(dirname(target), { recursive: true }) + await writeFile(target, bytes, { mode: stat.mode & 0o777 }) + files[file] = digest(bytes) + } + const before = await packages(source) + const pending = await intents(source) + const sourceSnapshot = digest(JSON.stringify(files)) + /** @type {Array<[string, string[]]>} */ + const steps = [ + ['git', ['init', '--quiet']], + [ + 'git', + [ + 'fetch', + '--quiet', + '--no-tags', + '--depth=1', + source, + `${revision}:refs/remotes/origin/main`, + ], + ], + ['git', ['update-ref', 'refs/heads/main', revision]], + ['git', ['symbolic-ref', 'HEAD', 'refs/heads/main']], + ['git', ['read-tree', revision]], + ['pnpm', ['install', '--frozen-lockfile', ...(offline ? ['--offline'] : [])]], + ['pnpm', ['release:version']], + ['pnpm', ['install', '--lockfile-only', '--ignore-scripts', ...(offline ? ['--offline'] : [])]], + ] + await mkdir(join(output, 'logs')) + for (let index = 0; index < steps.length; index++) { + const [command, args] = steps[index] + const text = await run(command, args, candidate) + await writeFile(join(output, 'logs', `${index + 1}-${command}.log`), String(text ?? '')) + } + const after = await packages(candidate) + const coreVersion = after.get('@inkcre/core')?.metadata.version + if (!semver.valid(coreVersion)) throw new Error('Candidate Core SDK version is invalid') + for (const [name, previous] of before) { + const next = after.get(name) + if (!next) throw new Error(`Candidate lost workspace package ${name}`) + if (JSON.stringify(previous.metadata.inkcre) !== JSON.stringify(next.metadata.inkcre)) + throw new Error(`Candidate changed native identity or Host SDK range for ${name}`) + const association = next.metadata.inkcre?.module_federation + if (association && !semver.satisfies(coreVersion, association.host_sdk_version)) + throw new Error( + `Candidate versions are not prepared: ${name} requires ${association.host_sdk_version}, Core is ${coreVersion}` + ) + } + const remaining = await intents(candidate) + if (remaining.length) + throw new Error(`Candidate still has unconsumed version intent: ${remaining.join(', ')}`) + await verifySource(source, files, run) + const summary = { + kind: 'isolated-native-validation', + source, + candidate, + source_revision: revision, + source_modified: sourceModified, + source_snapshot_sha256: sourceSnapshot, + prepared_at: new Date().toISOString(), + source_changesets: pending, + versions: Object.fromEntries( + [...after].map(([name, value]) => [ + name, + { + source: before.get(name)?.metadata.version, + prepared: value.metadata.version, + }, + ]) + ), + } + await writeFile( + join(output, 'source-snapshot.json'), + JSON.stringify({ ...summary, files }, null, 2) + '\n' + ) + await writeFile(join(output, 'candidate.json'), JSON.stringify(summary, null, 2) + '\n') + return summary +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const argument = (name) => process.argv[process.argv.indexOf(name) + 1] + if (!process.argv.includes('--output')) + throw new Error('Use --output ') + const summary = await prepareNativeCheckout({ + sourceDirectory: process.argv.includes('--source') ? argument('--source') : process.cwd(), + outputDirectory: argument('--output'), + offline: process.argv.includes('--offline'), + }) + console.log(JSON.stringify(summary)) +} diff --git a/scripts/sync-database-contract.mjs b/scripts/sync-database-contract.mjs index 7cd29b4e..801fa7e3 100644 --- a/scripts/sync-database-contract.mjs +++ b/scripts/sync-database-contract.mjs @@ -2,6 +2,7 @@ import { execFileSync } from 'node:child_process' import { createHash } from 'node:crypto' import { readFile, writeFile } from 'node:fs/promises' import { fileURLToPath } from 'node:url' +import { dirname, resolve } from 'node:path' import { inspectCoreRelease, @@ -17,19 +18,37 @@ import { stopDatabaseRuntime, waitForRuntime, } from './database-runtime-lib.mjs' -import { resolveDatabaseProviderConfig, runDatabaseDocker } from './database-provider.mjs' +import { + readExternalDatabaseDescriptor, + resolveDatabaseProviderConfig, + runDatabaseDocker, +} from './database-provider.mjs' +import { validateDevelopmentContract } from './database-contract-source.mjs' const repoRoot = fileURLToPath(new URL('..', import.meta.url)) const args = process.argv.slice(2) const imageIndex = args.indexOf('--image') const requestedImage = imageIndex >= 0 ? args[imageIndex + 1] : null +const runtimeIndex = args.indexOf('--runtime') +const contractIndex = args.indexOf('--runtime-contract') +const requestedRuntime = runtimeIndex >= 0 ? args[runtimeIndex + 1] : null +const requestedContract = contractIndex >= 0 ? args[contractIndex + 1] : null const check = args.includes('--check') +if ((runtimeIndex >= 0 || contractIndex >= 0) && (!requestedRuntime || !requestedContract)) { + throw new Error('local type generation requires --runtime and --runtime-contract together') +} +if (requestedRuntime && requestedImage) { + throw new Error('--runtime cannot be combined with --image') +} const config = validateCoreReleaseConfig(await readJson(`${repoRoot}/contracts/core-release.json`)) -const provider = resolveDatabaseProviderConfig() +const provider = requestedRuntime ? null : resolveDatabaseProviderConfig() const runDocker = (dockerArgs, options) => runDatabaseDocker(provider, dockerArgs, options) -const release = requestedImage - ? inspectCoreRelease(runDocker, requestedImage, { allowLocalImage: true }) - : resolveConfiguredCoreRelease(runDocker, config) +const release = requestedRuntime + ? null + : requestedImage + ? inspectCoreRelease(runDocker, requestedImage, { allowLocalImage: true }) + : resolveConfiguredCoreRelease(runDocker, config) +const sourceLabel = release?.image ?? 'explicit local development runtime' const instance = `typegen-${createHash('sha256') .update(`${repoRoot}/${process.pid}/${Date.now()}`) .digest('hex') @@ -78,20 +97,38 @@ function generateTypes(state, credentials) { async function writeOrCompare(path, content, label) { if (check) { const checked = await readFile(path) - if (!checked.equals(content)) throw new Error(`${label} is stale for ${release.image}`) + if (!checked.equals(content)) throw new Error(`${label} is stale for ${sourceLabel}`) return } await writeFile(path, content) } try { - process.env.INKCRE_CORE_IMAGE = release.image - const state = await ensureDatabaseRuntime(instance, { stdio: 'ignore' }) - runtimeStarted = true - await waitForRuntime(state) - const credentials = await runtimeCredentials(instance) + let state + let credentials + let compatibility + if (requestedRuntime) { + const descriptorPath = resolve(requestedRuntime) + state = readExternalDatabaseDescriptor({ descriptor: descriptorPath }) + const contract = await readJson(resolve(requestedContract)) + compatibility = validateDevelopmentContract(state, contract) + credentials = await readJson(`${dirname(descriptorPath)}/credential.json`) + } else { + process.env.INKCRE_CORE_IMAGE = release.image + state = await ensureDatabaseRuntime(instance, { stdio: 'ignore' }) + runtimeStarted = true + await waitForRuntime(state) + credentials = await runtimeCredentials(instance) + compatibility = projectDatabaseCompatibilityContract(release.runtime_contract) + } - const rawTypes = generateTypes(state, credentials) + let rawTypes + try { + rawTypes = generateTypes(state, credentials) + } catch { + // Child-process errors include argv, which contains the database password. + throw new Error('Supabase schema generation failed; inspect the development database owner') + } const generatedTypes = Buffer.from( output(workspaceBinary('oxfmt'), ['--stdin-filepath', 'database.generated.ts'], { input: rawTypes, @@ -104,7 +141,7 @@ try { ) const generatedRuntimeContract = Buffer.from( output(workspaceBinary('oxfmt'), ['--stdin-filepath', 'runtime-contract.generated.json'], { - input: stableJson(projectDatabaseCompatibilityContract(release.runtime_contract)), + input: stableJson(compatibility), }) ) await writeOrCompare( @@ -113,7 +150,7 @@ try { 'generated runtime contract' ) console.log( - `[contract] ${check ? 'verified' : 'synchronized'} ${release.image} through Supabase CLI` + `[contract] ${check ? 'verified' : 'synchronized'} ${sourceLabel} through Supabase CLI` ) } finally { if (runtimeStarted) { diff --git a/scripts/test-webext-e2e.mjs b/scripts/test-webext-e2e.mjs index 1f6100d2..de78ba4a 100644 --- a/scripts/test-webext-e2e.mjs +++ b/scripts/test-webext-e2e.mjs @@ -13,4 +13,11 @@ function run(command, args) { run('pnpm', ['--filter', '@inkcre/core', 'build']) run('pnpm', ['--filter', '@inkcre/webext', 'build']) +run('node', [ + '--test', + 'scripts/tests/browser-automation-transport.test.mjs', + 'scripts/tests/job-cancellation.test.mjs', + 'scripts/tests/source-anchor.test.mjs', +]) +run('node', ['scripts/verify-browser-automation.mjs']) run('pnpm', ['exec', 'playwright', 'test', '--project', 'browser-extension']) diff --git a/scripts/tests/browser-automation-transport.test.mjs b/scripts/tests/browser-automation-transport.test.mjs new file mode 100644 index 00000000..8b29ed59 --- /dev/null +++ b/scripts/tests/browser-automation-transport.test.mjs @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict' +import { createRequire } from 'node:module' +import { pathToFileURL } from 'node:url' +import test from 'node:test' + +const require = createRequire(new URL('../../packages/core/package.json', import.meta.url)) +const { ExtensionTransport } = await import( + pathToFileURL(require.resolve('puppeteer-core/internal/cdp/ExtensionTransport.js')).href +) + +void test('ExtensionTransport close awaits native detach and removes its event listener', async () => { + const original = globalThis.chrome + const listeners = new Set() + let completeDetach + const detached = new Promise((resolve) => { + completeDetach = resolve + }) + const calls = [] + globalThis.chrome = { + debugger: { + attach: async (target) => calls.push(['attach', target]), + detach: (target) => { + calls.push(['detach', target]) + return detached + }, + onEvent: { + addListener: (listener) => listeners.add(listener), + removeListener: (listener) => listeners.delete(listener), + }, + }, + } + try { + const transport = await ExtensionTransport.connectTab(42) + assert.equal(listeners.size, 1) + let finished = false + const closing = transport.close() + assert.ok(closing instanceof Promise) + void closing.then(() => { + finished = true + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(finished, false) + assert.equal(listeners.size, 0) + completeDetach() + await closing + assert.equal(finished, true) + assert.deepEqual(calls, [ + ['attach', { tabId: 42 }], + ['detach', { tabId: 42 }], + ]) + } finally { + globalThis.chrome = original + } +}) + +void test('ExtensionTransport close exposes detach failure to the session owner', async () => { + const original = globalThis.chrome + const listeners = new Set() + globalThis.chrome = { + debugger: { + attach: async () => undefined, + detach: async () => { + throw new Error('native detach failed') + }, + onEvent: { + addListener: (listener) => listeners.add(listener), + removeListener: (listener) => listeners.delete(listener), + }, + }, + } + try { + const transport = await ExtensionTransport.connectTab(1) + await assert.rejects(transport.close(), /native detach failed/) + assert.equal(listeners.size, 0) + } finally { + globalThis.chrome = original + } +}) diff --git a/scripts/tests/browser-automation/index.html b/scripts/tests/browser-automation/index.html new file mode 100644 index 00000000..1301179a --- /dev/null +++ b/scripts/tests/browser-automation/index.html @@ -0,0 +1,11 @@ + + + + + Browser automation acceptance + + +

Running browser automation acceptance…

+ + + diff --git a/scripts/tests/browser-automation/main.ts b/scripts/tests/browser-automation/main.ts new file mode 100644 index 00000000..efd45124 --- /dev/null +++ b/scripts/tests/browser-automation/main.ts @@ -0,0 +1,250 @@ +import { BrowserAccess, configStore, MetaConfigSchema } from '@inkcre/core' + +let stage = 'discover' + +function check(value: unknown, message: string): asserts value { + if (!value) throw new Error(message) +} + +async function main() { + if (location.pathname === '/unpaired' || location.pathname === '/paired') { + const found = await BrowserAccess.discover() + await fetch('/probe-report', { + method: 'POST', + body: JSON.stringify({ + token: new URLSearchParams(location.search).get('token'), + count: found.length, + providers: found.map((item) => item.provider), + }), + }) + return + } + const config = await fetch('/config').then((response) => response.json()) + configStore.metaConfig = MetaConfigSchema.parse({ + INKCRE_PGREST_URL: `${location.origin}/unused-database`, + INKCRE_PEER_ID: crypto.randomUUID(), + }) + const found = await BrowserAccess.discover() + check(found.length === 2, `Expected two authorized providers, found ${found.length}`) + const binding = found.find((item) => item.provider === config.provider)! + check(binding.provider === config.provider, 'Discovery returned another provider') + BrowserAccess.bind(binding) + check(BrowserAccess.hasBinding(binding.provider), 'Explicit binding was not persisted') + const baseline = await BrowserAccess.tabs(binding.provider) + let stalePage: { evaluate: (value: string) => Promise } | undefined + const extracted = await BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'new' } }, + async (page) => { + stalePage = page + stage = 'goto' + await page.goto(`${location.origin}/target`) + stage = 'wait-selector' + await page.waitForSelector('#more') + stage = 'read-account' + check( + await page.$eval('#account', (node) => node.textContent === 'fixture-account'), + 'Target lost login state' + ) + const response = page.waitForResponse((reply) => reply.url().endsWith('/data?page=2')) + stage = 'click-more' + await page.click('#more') + stage = 'response-json' + const body = await (await response).json() + stage = 'wait-later-batch' + await page.waitForFunction(() => document.querySelectorAll('li').length === 4, { + polling: 100, + }) + return { + body, + entries: await page.$$eval('li', (nodes) => nodes.map((node) => node.textContent)), + } + } + ) + check( + extracted.entries.join(',') === 'one,two,three,four', + 'Page interactions did not load the later batch' + ) + check(extracted.body.entries.length === 2, 'Network response body did not cross the bridge') + check( + (await BrowserAccess.tabs(binding.provider)).length === baseline.length, + 'Owned tab was not closed' + ) + check( + await stalePage!.evaluate('1').then( + () => false, + () => true + ), + 'Old Page still executed after release' + ) + + const borrowed = (await BrowserAccess.tabs(binding.provider)).find((tab) => + tab.url?.endsWith('/borrowed') + ) + check(borrowed, 'Fixture borrowed tab is missing') + stage = 'borrowed-tab' + await BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'existing', tab: borrowed } }, + async (page) => { + await page.evaluate(() => { + document.title = 'borrowed-preserved' + }) + const busyTab = (await BrowserAccess.tabs(binding.provider)).find( + (tab) => tab.tabId === borrowed.tabId + ) + check(busyTab, 'Active borrowed tab disappeared') + check( + await BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'existing', tab: busyTab } }, + async () => null + ).then( + () => false, + () => true + ), + 'Two sessions acquired the same borrowed tab' + ) + } + ) + check( + (await BrowserAccess.tabs(binding.provider)).some((tab) => tab.tabId === borrowed.tabId), + 'Borrowed tab was closed' + ) + check( + await BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'existing', tab: borrowed } }, + async () => null + ).then( + () => false, + () => true + ), + 'Consumed tab reference was reused' + ) + + const controller = new AbortController() + stage = 'abort' + let entered = false + const cancelled = BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'new' }, signal: controller.signal }, + async (page) => { + await page.goto(`${location.origin}/target`) + entered = true + setTimeout(() => controller.abort(), 50) + await page.waitForSelector('#never', { timeout: 0 }) + } + ) + check( + await cancelled.then( + () => false, + () => true + ), + 'Aborted callback completed successfully' + ) + check(entered, 'Abort did not exercise an active callback') + check( + (await BrowserAccess.tabs(binding.provider)).length === baseline.length, + 'Aborted owned tab was not cleaned up' + ) + + stage = 'provider-revoke-isolation' + const secondBinding = found.find((item) => item.provider === config.secondProvider) + check(secondBinding, 'Second provider is missing') + BrowserAccess.bind(secondBinding) + await BrowserAccess.withPage( + { provider: secondBinding.provider, target: { kind: 'new' } }, + async (page) => { + await page.goto(`${location.origin}/target`) + const revoked = await fetch('/fixture-command', { + method: 'POST', + body: JSON.stringify({ + type: 'browser-automation-revoke', + origin: location.origin, + provider: binding.provider, + }), + }).then((response) => response.json()) + check(revoked.ok, 'Native provider revocation failed') + check( + await page.$$eval('li', (nodes) => nodes.length === 2), + 'Revoking provider A disconnected provider B' + ) + } + ) + const wrongProvider = crypto.randomUUID() + BrowserAccess.bind({ ...binding, provider: wrongProvider }) + check( + await BrowserAccess.tabs(wrongProvider, AbortSignal.timeout(100)).then( + () => false, + () => true + ), + 'Another provider silently substituted for the exact target' + ) + BrowserAccess.unbind(wrongProvider) + const probe = async (route: string): Promise<{ count: number; providers: string[] }> => { + const token = crypto.randomUUID() + const opened = await fetch('/fixture-command', { + method: 'POST', + body: JSON.stringify({ + type: 'fixture-open-probe', + url: `${config.otherOrigin}${route}?token=${token}`, + }), + }).then((response) => response.json()) + check(opened.ok, 'Could not open a separate top-level Web origin') + const observed = await fetch(`/probe-result?token=${token}`).then((response) => response.json()) + await fetch('/fixture-command', { + method: 'POST', + body: JSON.stringify({ type: 'fixture-close-probe', tabId: opened.value.id }), + }) + return observed + } + check( + (await probe('/unpaired')).count === 0, + 'An unpaired origin on the same permitted host acquired a provider' + ) + const granted = await fetch('/fixture-command', { + method: 'POST', + body: JSON.stringify({ + type: 'browser-automation-grant', + origin: config.otherOrigin, + provider: secondBinding.provider, + }), + }).then((response) => response.json()) + check(granted.ok, 'Second origin could not be paired dynamically') + const paired = await probe('/paired') + check( + paired.count === 1 && paired.providers[0] === secondBinding.provider, + 'Second origin did not discover its exact authorized provider' + ) + await fetch('/report', { + method: 'POST', + body: JSON.stringify({ + ok: true, + entries: extracted.entries, + cases: [ + 'provider-revoke-isolation', + 'second-origin-pairing', + 'login-state', + 'page-navigation', + 'dynamic-batch', + 'network-json', + 'new-tab-cleanup', + 'borrowed-tab-preserved', + 'borrowed-tab-exclusion', + 'stale-page', + 'stale-tab-reference', + 'abort-cleanup', + 'exact-provider', + 'exact-origin', + ], + }), + }) + document.querySelector('#status')!.textContent = 'Passed' +} + +void main().catch(async (error: unknown) => { + document.querySelector('#status')!.textContent = String(error) + await fetch('/report', { + method: 'POST', + body: JSON.stringify({ + ok: false, + error: `${stage}: ${error instanceof Error ? `${error.name}: ${error.message}\n${error.stack}` : String(error)}`, + }), + }) +}) diff --git a/scripts/tests/database-contract-source.test.mjs b/scripts/tests/database-contract-source.test.mjs new file mode 100644 index 00000000..bfc54cac --- /dev/null +++ b/scripts/tests/database-contract-source.test.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { validateDevelopmentContract } from '../database-contract-source.mjs' + +const descriptor = { + profile: 'development', + owner_repository: 'InKCre/impl-py', + contract_revision: 'source-batch-persistence-v1', + migration_head: 'batch-head', +} +const contract = { + format: 1, + revision: descriptor.contract_revision, + migration_heads: ['batch-head'], + protocol: { format: 1, schema: 'inkcre' }, + jwt: { algorithm: 'HS256', role: 'authenticated', issuer: 'inkcre-peer', audience: 'inkcre-api' }, +} + +void test('local type generation projects the matching owner contract without release provenance', () => { + assert.deepEqual(validateDevelopmentContract(descriptor, contract), { + format: 1, + revision: descriptor.contract_revision, + protocol: contract.protocol, + jwt: contract.jwt, + }) +}) + +void test('local type generation rejects mismatched owner state and source contracts', () => { + for (const change of [ + { profile: 'production' }, + { owner_repository: 'other' }, + { converging: true }, + { contract_revision: 'old-revision' }, + { migration_head: 'old-head' }, + ]) { + assert.throws(() => validateDevelopmentContract({ ...descriptor, ...change }, contract)) + } +}) diff --git a/scripts/tests/job-cancellation.test.mjs b/scripts/tests/job-cancellation.test.mjs new file mode 100644 index 00000000..3e1640a1 --- /dev/null +++ b/scripts/tests/job-cancellation.test.mjs @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { Job, JobManager, JobStatus, MetaConfigSchema } from '../../packages/core/dist/index.js' + +function pending(id, type, abortRequested) { + return Job.parse({ + id, + type, + parameters: {}, + state: {}, + abort_requested: abortRequested, + timeout_seconds: 30, + status: JobStatus.PENDING, + created_at: new Date(), + started_at: null, + closed_at: null, + }) +} + +function database(t, job, abortWhenClaimed) { + const updates = [] + t.mock.method(Job, 'get', async () => job) + t.mock.method(Job.dbApi, 'update', (fields) => ({ + eq() { + return this + }, + select() { + if (fields.status === JobStatus.RUNNING) + return { + maybeSingle: async () => ({ + data: { ...job, status: JobStatus.RUNNING, abort_requested: abortWhenClaimed }, + }), + } + updates.push(fields.status) + return Promise.resolve({ data: [{ id: job.id }] }) + }, + })) + return updates +} + +void test('an already cancelled pending Job closes without asking business eligibility or running', async (t) => { + const type = `test.cancel.${crypto.randomUUID()}` + const job = pending(1, type, true) + const updates = database(t, job, true) + const canHandle = t.mock.fn(() => false) + const handle = t.mock.fn(async () => undefined) + JobManager.registerHandler(type, { parameters: MetaConfigSchema, canHandle, handle }) + try { + assert.equal(await JobManager.run(job.id), true) + assert.equal(canHandle.mock.callCount(), 0) + assert.equal(handle.mock.callCount(), 0) + assert.deepEqual(updates, [JobStatus.ABORTED]) + } finally { + JobManager.unregisterHandler(type) + } +}) + +void test('a cancellation arriving during atomic claim prevents the handler from starting', async (t) => { + const type = `test.cancel.${crypto.randomUUID()}` + const job = pending(2, type, false) + const updates = database(t, job, true) + const handle = t.mock.fn(async () => undefined) + JobManager.registerHandler(type, { parameters: MetaConfigSchema, canHandle: () => true, handle }) + try { + assert.equal(await JobManager.run(job.id), true) + assert.equal(handle.mock.callCount(), 0) + assert.deepEqual(updates, [JobStatus.ABORTED]) + } finally { + JobManager.unregisterHandler(type) + } +}) + +void test('ordinary ineligible work remains pending', async (t) => { + const type = `test.cancel.${crypto.randomUUID()}` + const job = pending(3, type, false) + const updates = database(t, job, false) + const handle = t.mock.fn(async () => undefined) + JobManager.registerHandler(type, { parameters: MetaConfigSchema, canHandle: () => false, handle }) + try { + assert.equal(await JobManager.run(job.id), false) + assert.equal(handle.mock.callCount(), 0) + assert.deepEqual(updates, []) + } finally { + JobManager.unregisterHandler(type) + } +}) diff --git a/scripts/tests/prepare-native-checkout.test.mjs b/scripts/tests/prepare-native-checkout.test.mjs new file mode 100644 index 00000000..63eb575b --- /dev/null +++ b/scripts/tests/prepare-native-checkout.test.mjs @@ -0,0 +1,187 @@ +import assert from 'node:assert/strict' +import { mkdtemp, mkdir, readFile, rm, writeFile, unlink } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { prepareNativeCheckout } from '../prepare-native-checkout.mjs' + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), 'inkcre-native-checkout-')) + const source = join(root, 'source') + const output = join(root, 'candidate') + const input = { + 'package.json': { name: 'fixture', private: true }, + 'packages/core/package.json': { name: '@inkcre/core', version: '0.4.3' }, + 'extensions/example/package.json': { + name: '@inkcre/example', + version: '0.1.0', + inkcre: { + name: 'inkcre/example', + module_federation: { host_sdk: '@inkcre/core', host_sdk_version: '>=0.4.4 <0.5.0' }, + }, + }, + '.changeset/compatible.md': '---\n"@inkcre/core": patch\n---\nCompatible capability\n', + } + for (const [file, value] of Object.entries(input)) { + await mkdir(join(source, file, '..'), { recursive: true }) + await writeFile(join(source, file), typeof value === 'string' ? value : JSON.stringify(value)) + } + const calls = [] + const run = async (command, args, cwd) => { + calls.push({ command, args, cwd }) + if (command === 'git' && args[0] === 'rev-parse') return 'a'.repeat(40) + if (command === 'git' && args[0] === 'status') return ' M packages/core/package.json\n' + if (command === 'git' && args[0] === 'ls-files') return Object.keys(input).join('\0') + '\0' + if (command === 'pnpm' && args[0] === 'release:version') { + await writeFile( + join(cwd, 'packages/core/package.json'), + JSON.stringify({ name: '@inkcre/core', version: '0.4.4' }) + ) + await unlink(join(cwd, '.changeset/compatible.md')) + } + return '' + } + return { root, source, output, calls, run, input } +} + +void test('the existing version command prepares an isolated candidate without consuming source intent', async () => { + const context = await fixture() + try { + const result = await prepareNativeCheckout({ + sourceDirectory: context.source, + outputDirectory: context.output, + run: context.run, + offline: true, + }) + assert.equal(result.versions['@inkcre/core'].prepared, '0.4.4') + assert.equal( + JSON.parse(await readFile(join(context.source, 'packages/core/package.json'), 'utf8')) + .version, + '0.4.3' + ) + assert.match( + await readFile(join(context.source, '.changeset/compatible.md'), 'utf8'), + /Compatible capability/ + ) + assert.ok( + context.calls.some( + (call) => + call.command === 'pnpm' && + call.args[0] === 'release:version' && + call.cwd === result.candidate + ) + ) + assert.ok( + context.calls.some((call) => call.command === 'pnpm' && call.args.includes('--offline')) + ) + assert.equal( + context.calls.some((call) => call.args[0] === 'commit'), + false + ) + assert.equal(result.source_modified, true) + } finally { + await rm(context.root, { recursive: true, force: true }) + } +}) + +void test('a version command that leaves the Host below the producer minimum is rejected', async () => { + const context = await fixture() + try { + const run = async (command, args, cwd) => { + if (command === 'pnpm' && args[0] === 'release:version') { + await unlink(join(cwd, '.changeset/compatible.md')) + return '' + } + return context.run(command, args, cwd) + } + await assert.rejects( + prepareNativeCheckout({ + sourceDirectory: context.source, + outputDirectory: context.output, + run, + }), + /versions are not prepared/ + ) + } finally { + await rm(context.root, { recursive: true, force: true }) + } +}) + +void test('loosening a future producer range cannot masquerade as version preparation', async () => { + const context = await fixture() + try { + const run = async (command, args, cwd) => { + const result = await context.run(command, args, cwd) + if (command === 'pnpm' && args[0] === 'release:version') { + const file = join(cwd, 'extensions/example/package.json') + const value = JSON.parse(await readFile(file, 'utf8')) + value.inkcre.module_federation.host_sdk_version = '>=0.4.3 <0.5.0' + await writeFile(file, JSON.stringify(value)) + } + return result + } + await assert.rejects( + prepareNativeCheckout({ + sourceDirectory: context.source, + outputDirectory: context.output, + run, + }), + /changed native identity or Host SDK range/ + ) + } finally { + await rm(context.root, { recursive: true, force: true }) + } +}) + +void test('source changes during preparation invalidate the candidate without reverting the writer', async () => { + const context = await fixture() + try { + const run = async (command, args, cwd) => { + const result = await context.run(command, args, cwd) + if (command === 'pnpm' && args[0] === 'release:version') + await writeFile(join(context.source, '.changeset/compatible.md'), 'concurrent source work') + return result + } + await assert.rejects( + prepareNativeCheckout({ + sourceDirectory: context.source, + outputDirectory: context.output, + run, + }), + /Source changed after snapshot/ + ) + assert.equal( + await readFile(join(context.source, '.changeset/compatible.md'), 'utf8'), + 'concurrent source work' + ) + } finally { + await rm(context.root, { recursive: true, force: true }) + } +}) + +void test('new source files added during preparation require a fresh snapshot too', async () => { + const context = await fixture() + try { + const run = async (command, args, cwd) => { + const result = await context.run(command, args, cwd) + if (command === 'pnpm' && args[0] === 'release:version') { + context.input['extensions/example/new-source.ts'] = 'export const added = true\n' + await writeFile( + join(context.source, 'extensions/example/new-source.ts'), + context.input['extensions/example/new-source.ts'] + ) + } + return result + } + await assert.rejects( + prepareNativeCheckout({ + sourceDirectory: context.source, + outputDirectory: context.output, + run, + }), + /Source file set changed after snapshot/ + ) + } finally { + await rm(context.root, { recursive: true, force: true }) + } +}) diff --git a/scripts/tests/source-anchor.test.mjs b/scripts/tests/source-anchor.test.mjs new file mode 100644 index 00000000..02b50781 --- /dev/null +++ b/scripts/tests/source-anchor.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { Block, SourceResolver, ResolverContentError } from '../../packages/core/dist/index.js' + +const anchor = (content) => + Block.parse({ + id: 41, + resolver: 'core.source.v1', + storage: null, + content: JSON.stringify(content), + }) + +void test('Source anchors read canonical identity without needing the collector or its config', async () => { + const resolver = new SourceResolver( + anchor({ id: 9, type: 'extensions.bilibili.favorites.v1', nickname: '我的收藏' }) + ) + assert.equal( + await resolver.getText(), + 'Source 9 named 我的收藏 uses extensions.bilibili.favorites.v1.' + ) + assert.equal(await resolver.getSolvedContent(), await resolver.getText()) + assert.equal( + await new SourceResolver(anchor({ id: 10, type: 'source.example' })).getText(), + 'Source 10 uses source.example.' + ) +}) + +void test('Source anchor decoding rejects a parallel config or malformed identity', async () => { + for (const content of [ + { id: '9', type: 'source.example' }, + { id: 9, type: 'source.example', config: { cookie: 'must-not-be-an-anchor' } }, + { id: 9 }, + ]) { + await assert.rejects(new SourceResolver(anchor(content)).getText(), ResolverContentError) + } +}) diff --git a/scripts/verify-browser-automation.mjs b/scripts/verify-browser-automation.mjs new file mode 100644 index 00000000..6f33c042 --- /dev/null +++ b/scripts/verify-browser-automation.mjs @@ -0,0 +1,213 @@ +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { createServer } from 'node:http' +import { createRequire } from 'node:module' +import { once } from 'node:events' +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { join, extname } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { chromium } from '@playwright/test' + +const root = fileURLToPath(new URL('../', import.meta.url)) +const require = createRequire(new URL('../apps/client-web/package.json', import.meta.url)) +const { build } = await import(pathToFileURL(require.resolve('vite')).href) +await mkdir(join(root, '.runtime'), { recursive: true }) +const temporary = await mkdtemp(join(root, '.runtime/browser-automation-')) +const dist = join(temporary, 'web') +const extension = join(temporary, 'extension') +const provider = randomUUID() +const secondProvider = randomUUID() +const nativeCommands = [] +const nativeReplies = new Map() +const probeResults = new Map() +const probeWaiters = new Map() +let appOrigin +let otherOrigin +let webReport +let nativeReport +let done +const finished = new Promise((resolve) => { + done = resolve +}) + +async function handle(request, response) { + const url = new URL(request.url, appOrigin ?? 'http://127.0.0.1') + if (url.pathname === '/config') + return response.end(JSON.stringify({ provider, secondProvider, otherOrigin })) + if (url.pathname === '/state') return response.end(JSON.stringify({ complete: !!webReport })) + if (url.pathname === '/fixture-next') + return response.end(JSON.stringify(nativeCommands.shift() ?? null)) + if (url.pathname === '/probe-result') { + const token = url.searchParams.get('token') + if (probeResults.has(token)) return response.end(JSON.stringify(probeResults.get(token))) + probeWaiters.set(token, (result) => response.end(JSON.stringify(result))) + return + } + if (url.pathname === '/probe-report') { + let body = '' + for await (const chunk of request) body += chunk + const result = JSON.parse(body) + probeResults.set(result.token, result) + probeWaiters.get(result.token)?.(result) + probeWaiters.delete(result.token) + return response.end('ok') + } + if (url.pathname === '/fixture-command') { + let body = '' + for await (const chunk of request) body += chunk + const id = randomUUID() + nativeCommands.push({ id, message: JSON.parse(body) }) + nativeReplies.set(id, (reply) => response.end(JSON.stringify(reply))) + return + } + if (url.pathname === '/fixture-reply') { + let body = '' + for await (const chunk of request) body += chunk + const { id, reply } = JSON.parse(body) + nativeReplies.get(id)?.(reply) + nativeReplies.delete(id) + return response.end('ok') + } + if (url.pathname === '/report' || url.pathname === '/native-report') { + let body = '' + for await (const chunk of request) body += chunk + if (url.pathname === '/report') webReport = JSON.parse(body) + else nativeReport = JSON.parse(body) + response.end('ok') + if (webReport && nativeReport) done() + return + } + if (url.pathname === '/borrowed') + return response.end('borrowed

Borrowed tab

') + if (url.pathname === '/target') { + response.setHeader('Content-Type', 'text/html') + return response.end( + `

${request.headers.cookie?.includes('fixture-login=1') ? 'fixture-account' : 'not-logged-in'}

  • one
  • two
` + ) + } + if (url.pathname === '/data') { + response.setHeader('Content-Type', 'application/json') + return response.end(JSON.stringify({ entries: ['three', 'four'] })) + } + const path = url.pathname.startsWith('/assets/') + ? join(dist, url.pathname) + : join(dist, 'index.html') + try { + response.setHeader( + 'Content-Type', + extname(path) === '.js' + ? 'application/javascript' + : extname(path) === '.css' + ? 'text/css' + : 'text/html' + ) + response.setHeader('Set-Cookie', 'fixture-login=1; SameSite=Lax; Path=/') + response.end(await readFile(path)) + } catch { + response.writeHead(404).end() + } +} +const servers = [ + createServer((request, response) => void handle(request, response)), + createServer((request, response) => void handle(request, response)), +] +for (const server of servers) server.listen(0, '127.0.0.1') +await Promise.all(servers.map((server) => once(server, 'listening'))) +appOrigin = `http://127.0.0.1:${servers[0].address().port}` +otherOrigin = `http://127.0.0.1:${servers[1].address().port}` +let browserProcess +let stderr = '' +let timeout +try { + await build({ + configFile: false, + root: join(root, 'scripts/tests/browser-automation'), + resolve: { alias: { '@inkcre/core': join(root, 'packages/core/src/index.ts') } }, + build: { + outDir: dist, + emptyOutDir: true, + }, + }) + await cp(join(root, 'apps/client-webext/.output/chrome-mv3'), extension, { recursive: true }) + const manifest = JSON.parse(await readFile(join(extension, 'manifest.json'), 'utf8')) + // This isolated fixture pregrants only its synthetic host. Native permission UX is separate evidence. + manifest.host_permissions = ['http://127.0.0.1/*'] + manifest.background.service_worker = 'automation-fixture-background.js' + manifest.background.type = 'module' + await writeFile(join(extension, 'manifest.json'), JSON.stringify(manifest)) + await writeFile( + join(extension, 'automation-fixture.html'), + '' + ) + await writeFile( + join(extension, 'automation-fixture-background.js'), + "import './background.js'; chrome.runtime.onInstalled.addListener(() => { void chrome.tabs.create({url: chrome.runtime.getURL('automation-fixture.html')}); });" + ) + await writeFile( + join(extension, 'automation-fixture.js'), + ` +(async()=>{ +const origin=${JSON.stringify(appOrigin)},provider=${JSON.stringify(provider)},secondProvider=${JSON.stringify(secondProvider)}; +const db=await new Promise((resolve,reject)=>{const r=indexedDB.open('inkcre-webext-collection',1);r.onupgradeneeded=()=>{for(const name of ['settings','connections','outbox','imports','legacy'])if(!r.result.objectStoreNames.contains(name))r.result.createObjectStore(name)};r.onsuccess=()=>resolve(r.result);r.onerror=()=>reject(r.error)}); +await new Promise((resolve,reject)=>{const tx=db.transaction(['connections'],'readwrite');for(const peerId of [provider,secondProvider]){const id=crypto.randomUUID();tx.objectStore('connections').put({id,name:'Fixture browser',url:origin+'/unused-database',secret:'fixture-only-not-a-real-secret-32-bytes',peerId,storageId:-4},id)}tx.oncomplete=resolve;tx.onerror=()=>reject(tx.error)}); +let result; +for(let i=0;i<30;i++){try{result=await chrome.runtime.sendMessage({type:'browser-automation-grant',origin,provider});if(result?.ok)break}catch{}await new Promise(r=>setTimeout(r,100))} +if(!result?.ok)throw new Error(result?.error??'Background did not initialize'); +const second=await chrome.runtime.sendMessage({type:'browser-automation-grant',origin,provider:secondProvider});if(!second?.ok)throw new Error(second?.error??'Second provider grant failed'); +await chrome.tabs.create({url:origin+'/borrowed',active:false}); +await chrome.tabs.create({url:origin+'/app',active:false}); +for(let i=0;i<1200;i++){const command=await fetch(origin+'/fixture-next').then(r=>r.json());if(command){const reply=command.message.type==='fixture-open-probe'?{ok:true,value:await chrome.tabs.create({url:command.message.url,active:false})}:command.message.type==='fixture-close-probe'?{ok:true,value:await chrome.tabs.remove(command.message.tabId)}:await chrome.runtime.sendMessage(command.message);await fetch(origin+'/fixture-reply',{method:'POST',body:JSON.stringify({id:command.id,reply})})}if((await fetch(origin+'/state').then(r=>r.json())).complete){const targets=await chrome.debugger.getTargets();await fetch(origin+'/native-report',{method:'POST',body:JSON.stringify({attached:targets.filter(t=>t.attached).map(t=>({id:t.id,tabId:t.tabId,url:t.url}))})});return}await new Promise(r=>setTimeout(r,100))} +throw new Error('Web fixture timed out'); +})().catch(async error=>{await fetch(${JSON.stringify(appOrigin)}+'/report',{method:'POST',body:JSON.stringify({ok:false,error:error.stack})});await fetch(${JSON.stringify(appOrigin)}+'/native-report',{method:'POST',body:'{}'})}); +` + ) + const executable = globalThis.process.env.INKCRE_CHROMIUM_BINARY || chromium.executablePath() + browserProcess = spawn( + executable, + [ + '--headless=new', + '--no-first-run', + '--no-default-browser-check', + '--disable-background-networking', + `--user-data-dir=${join(temporary, 'profile')}`, + `--disable-extensions-except=${extension}`, + `--load-extension=${extension}`, + '--enable-logging=stderr', + 'about:blank', + ], + { stdio: ['ignore', 'ignore', 'pipe'] } + ) + browserProcess.stderr.on('data', (chunk) => { + stderr = (stderr + chunk).slice(-4000) + }) + browserProcess.on('error', (error) => { + webReport = { ok: false, error: String(error) } + done() + }) + await Promise.race([ + finished, + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(`Browser fixture timed out\n${stderr}`)), 120_000) + }), + ]) + assert.equal(webReport?.ok, true, webReport?.error ?? stderr) + assert.deepEqual( + nativeReport?.attached, + [], + 'An owned debugger attachment survived callback cleanup' + ) + console.log(JSON.stringify({ browser: executable, ...webReport, nativeReport }, null, 2)) +} finally { + clearTimeout(timeout) + if (browserProcess && browserProcess.exitCode === null) { + browserProcess.kill('SIGTERM') + await Promise.race([ + once(browserProcess, 'exit'), + new Promise((resolve) => setTimeout(resolve, 5000)), + ]) + if (browserProcess.exitCode === null) browserProcess.kill('SIGKILL') + } + for (const server of servers) server.close() + await rm(temporary, { recursive: true, force: true }) +} diff --git a/scripts/webext-release.mjs b/scripts/webext-release.mjs index 812be437..2c3a1576 100644 --- a/scripts/webext-release.mjs +++ b/scripts/webext-release.mjs @@ -35,7 +35,8 @@ export function checkManifest(manifest, version, browser, key) { if (browser === 'chrome') assert.equal(manifest.key, key, 'Release must use the fixed Chromium identity') else assert.equal(manifest.browser_specific_settings?.gecko?.id, 'webext@inkcre.org') - assert.ok(!manifest.permissions.includes('debugger')) + assert.equal(manifest.permissions.includes('debugger'), browser === 'chrome') + if (browser === 'chrome') assert.equal(manifest.minimum_chrome_version, '125') } async function api(route, options = {}) { diff --git a/svc.json b/svc.json index b76509d4..b8f07cd1 100644 --- a/svc.json +++ b/svc.json @@ -1,84 +1,144 @@ { - "schema_version": 4, + "schema_version": 5, "dev": { - "targets": { + "resources": { "database": { - "poll_interval": 1, - "probe": { - "argv": ["node", "scripts/probe-database.mjs", "${dev.instance}"], - "cwd": ".", - "kind": "exec", - "output_limit": 2048, - "timeout": 5 + "scope": "worktree", + "observations": { + "ready": { + "label": "就绪状态", + "purpose": "runtime-readiness", + "probe": { + "argv": ["node", "scripts/probe-database.mjs", "${dev.instance}"], + "cwd": ".", + "kind": "exec", + "output_limit": 2048, + "timeout": 5 + } + } }, - "provision": { - "argv": ["node", "scripts/database-runtime.mjs", "ensure", "${dev.instance}"], - "cwd": ".", - "env": { - "INKCRE_DATABASE_PROVIDER": "local" - }, - "kind": "exec", - "mode": "run" - }, - "readiness_timeout": 180, - "scope": "worktree" + "actions": { + "ensure": { + "argv": ["node", "scripts/database-runtime.mjs", "ensure", "${dev.instance}"], + "cwd": ".", + "env": { + "INKCRE_DATABASE_PROVIDER": "local" + }, + "kind": "exec", + "intent": "准备database开发资源", + "traits": ["runtime", "provision"], + "effect_scope": "当前工作树的database开发资源", + "data_effect": "modify", + "completion": "exit", + "interruption": "wait", + "conditions": ["ready"], + "completion_timeout": 180, + "poll_interval": 1, + "convergent": true + } + } }, "web": { - "access": ["https://client-web-${dev.instance}.localhost/"], - "poll_interval": 0.5, - "probe": { - "argv": ["node", "scripts/probe-dev.mjs", "web", "${dev.instance}"], - "cwd": ".", - "kind": "exec", - "output_limit": 1024, - "timeout": 5 + "scope": "worktree", + "observations": { + "ready": { + "label": "就绪状态", + "purpose": "runtime-readiness", + "probe": { + "argv": ["node", "scripts/probe-dev.mjs", "web", "${dev.instance}"], + "cwd": ".", + "kind": "exec", + "output_limit": 1024, + "timeout": 5 + } + } }, - "provision": { - "argv": ["node", "scripts/dev.mjs", "web"], - "cwd": ".", - "kind": "exec", - "mode": "run" + "actions": { + "ensure": { + "argv": ["node", "scripts/dev.mjs", "web"], + "cwd": ".", + "kind": "exec", + "intent": "准备web开发资源", + "traits": ["runtime", "provision"], + "effect_scope": "当前工作树的web开发资源", + "data_effect": "modify", + "completion": "ready", + "interruption": "terminate", + "conditions": ["ready"], + "completion_timeout": 90, + "poll_interval": 0.5, + "convergent": true + } }, - "readiness_timeout": 90, - "scope": "worktree" + "access": ["https://client-web-${dev.instance}.localhost/"] }, "web-ui": { - "access": ["https://client-web-ui-${dev.instance}.localhost/"], - "poll_interval": 0.5, - "probe": { - "argv": ["node", "scripts/probe-dev.mjs", "web-ui", "${dev.instance}"], - "cwd": ".", - "kind": "exec", - "output_limit": 1024, - "timeout": 5 + "scope": "worktree", + "observations": { + "ready": { + "label": "就绪状态", + "purpose": "runtime-readiness", + "probe": { + "argv": ["node", "scripts/probe-dev.mjs", "web-ui", "${dev.instance}"], + "cwd": ".", + "kind": "exec", + "output_limit": 1024, + "timeout": 5 + } + } }, - "provision": { - "argv": ["node", "scripts/dev.mjs", "web-ui"], - "cwd": ".", - "kind": "exec", - "mode": "run" + "actions": { + "ensure": { + "argv": ["node", "scripts/dev.mjs", "web-ui"], + "cwd": ".", + "kind": "exec", + "intent": "准备web-ui开发资源", + "traits": ["runtime", "provision"], + "effect_scope": "当前工作树的web-ui开发资源", + "data_effect": "modify", + "completion": "ready", + "interruption": "terminate", + "conditions": ["ready"], + "completion_timeout": 90, + "poll_interval": 0.5, + "convergent": true + } }, - "readiness_timeout": 90, - "scope": "worktree" + "access": ["https://client-web-ui-${dev.instance}.localhost/"] }, "webext": { - "access": ["https://webext-${dev.instance}.localhost/popup.html"], - "poll_interval": 0.5, - "probe": { - "argv": ["node", "scripts/probe-dev.mjs", "webext", "${dev.instance}"], - "cwd": ".", - "kind": "exec", - "output_limit": 1024, - "timeout": 5 + "scope": "worktree", + "observations": { + "ready": { + "label": "就绪状态", + "purpose": "runtime-readiness", + "probe": { + "argv": ["node", "scripts/probe-dev.mjs", "webext", "${dev.instance}"], + "cwd": ".", + "kind": "exec", + "output_limit": 1024, + "timeout": 5 + } + } }, - "provision": { - "argv": ["node", "scripts/dev.mjs", "webext"], - "cwd": ".", - "kind": "exec", - "mode": "run" + "actions": { + "ensure": { + "argv": ["node", "scripts/dev.mjs", "webext"], + "cwd": ".", + "kind": "exec", + "intent": "准备webext开发资源", + "traits": ["runtime", "provision"], + "effect_scope": "当前工作树的webext开发资源", + "data_effect": "modify", + "completion": "ready", + "interruption": "terminate", + "conditions": ["ready"], + "completion_timeout": 120, + "poll_interval": 0.5, + "convergent": true + } }, - "readiness_timeout": 120, - "scope": "worktree" + "access": ["https://webext-${dev.instance}.localhost/popup.html"] } } } diff --git a/tests/e2e/browser-extension.spec.ts b/tests/e2e/browser-extension.spec.ts index 957abe0e..54399d65 100644 --- a/tests/e2e/browser-extension.spec.ts +++ b/tests/e2e/browser-extension.spec.ts @@ -680,7 +680,7 @@ test('settings preserve the WebExt card style across desktop, narrow and dark la ) await expect(page.getByLabel('PostgREST 地址')).toBeVisible() const layout = await page.evaluate(() => { - const card = document.querySelector('.collection-page > .section')! + const card = document.querySelector('section[aria-labelledby="connection-heading"]')! const input = card.querySelector('input')! const primary = card.querySelector('button[type="submit"]')! const toggle = document.querySelector('[role="switch"]')! From 7dd495c902c72746bcbc9d0d9b2878d2e895f307 Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 00:11:43 +0800 Subject: [PATCH 02/12] =?UTF-8?q?fix(release):=20=E5=AF=B9=E9=BD=90?= =?UTF-8?q?=E6=96=B0=20SDK=20=E5=9F=BA=E7=BA=BF=E4=B8=8E=E9=9A=94=E7=A6=BB?= =?UTF-8?q?=E5=80=99=E9=80=89=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将新 B 站分发限定到尚待发布的 Core 0.4.5，沿用应用重命名后的产物路径 - 保留独立 SDK 构建图与观测检查，Pages preview 复用正式版本准备入口 - 不改变数据库生产准入引用，保留跨仓协议发布依赖 --- .changeset/web-source-browser-automation.md | 3 ++- .github/preview/extensions.json | 2 +- .github/workflows/ci.yml | 5 +++++ .github/workflows/pages-preview.yml | 16 +++++++++++----- docs/40-deployment/native-extension-delivery.md | 6 +++++- extensions/bilibili/package.json | 2 +- extensions/bilibili/tsconfig.json | 2 +- extensions/bilibili/vite.config.ts | 2 +- scripts/tests/prepare-native-checkout.test.mjs | 12 ++++++------ scripts/verify-browser-automation.mjs | 4 ++-- 10 files changed, 35 insertions(+), 19 deletions(-) diff --git a/.changeset/web-source-browser-automation.md b/.changeset/web-source-browser-automation.md index 3d42d459..388b6285 100644 --- a/.changeset/web-source-browser-automation.md +++ b/.changeset/web-source-browser-automation.md @@ -1,6 +1,7 @@ --- '@inkcre/core': patch -'@inkcre/client-webext': patch +'@inkcre/web': patch +'@inkcre/webext': patch --- 为 Web Source 提供基于 Puppeteer 的本机浏览器能力，支持明确的 Web origin 授权、精确浏览器绑定、登录态页面交互与网络响应读取。 diff --git a/.github/preview/extensions.json b/.github/preview/extensions.json index 903ce8d6..d7158886 100644 --- a/.github/preview/extensions.json +++ b/.github/preview/extensions.json @@ -4,7 +4,7 @@ { "kind": "module_federation", "producer": "../../extensions/bilibili/package.json", - "artifact": "../../extensions/bilibili/dist/client-web" + "artifact": "../../extensions/bilibili/dist/web" }, { "kind": "module_federation", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5b4e081..170a1430 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,6 +120,10 @@ jobs: working-directory: ${{ runner.temp }}/native-checkout/workspace run: pnpm build + - name: Verify independent SDK and Job observations + working-directory: ${{ runner.temp }}/native-checkout/workspace + run: pnpm check:sdk && pnpm check:web + - name: 验证 B 站收藏采集规则 working-directory: ${{ runner.temp }}/native-checkout/workspace run: pnpm --filter @inkcre/ext-bilibili test @@ -136,6 +140,7 @@ jobs: scripts/tests/job-cancellation.test.mjs scripts/tests/source-anchor.test.mjs scripts/tests/prepare-native-checkout.test.mjs + scripts/tests/database-contract-source.test.mjs peer-database: # Existing branch protection binds this exact check name. diff --git a/.github/workflows/pages-preview.yml b/.github/workflows/pages-preview.yml index dcd3e625..e481f4e3 100644 --- a/.github/workflows/pages-preview.yml +++ b/.github/workflows/pages-preview.yml @@ -102,8 +102,14 @@ jobs: env: PDM_CHECK_UPDATE: 'false' + - name: Prepare isolated preview versions with the release command + run: >- + node candidate/scripts/prepare-native-checkout.mjs + --source candidate + --output "$RUNNER_TEMP/pages-native-checkout" + - name: Build the exact-head SPA and Module Federation snapshots - working-directory: candidate + working-directory: ${{ runner.temp }}/pages-native-checkout/workspace run: pnpm build - name: Build the same-origin Preview Registry facade @@ -112,11 +118,11 @@ jobs: PREVIEW_ORIGIN: https://preview-client-web-pr-${{ needs.identity.outputs.pull_number }}.${{ vars.CLOUDFLARE_PAGES_PROJECT }}.pages.dev run: | pdm run inkcre-ext preview build \ - --inventory .github/preview/extensions.json \ + --inventory "$RUNNER_TEMP/pages-native-checkout/workspace/.github/preview/extensions.json" \ --public-origin "$PREVIEW_ORIGIN" \ - --output .preview-registry - cp -R .preview-registry/. apps/web/dist/ - cp -R apps/web/dist "$GITHUB_WORKSPACE/.pages-preview" + --output "$RUNNER_TEMP/pages-native-checkout/workspace/.preview-registry" + cp -R "$RUNNER_TEMP/pages-native-checkout/workspace/.preview-registry/." "$RUNNER_TEMP/pages-native-checkout/workspace/apps/web/dist/" + cp -R "$RUNNER_TEMP/pages-native-checkout/workspace/apps/web/dist" "$GITHUB_WORKSPACE/.pages-preview" - name: Bind Module Federation assets to the selected snapshot env: diff --git a/docs/40-deployment/native-extension-delivery.md b/docs/40-deployment/native-extension-delivery.md index 1b2db40d..4bed95dd 100644 --- a/docs/40-deployment/native-extension-delivery.md +++ b/docs/40-deployment/native-extension-delivery.md @@ -73,9 +73,13 @@ changesets in the PR nor replaces the normal Version PR and protected-main relea Workspace contract 先在原 checkout 校验格式、静态规则和类型，再通过 `scripts/prepare-native-checkout.mjs` 将当前未忽略的源文件复制到隔离候选，运行正式 -`pnpm release:version`，在候选中完成生产构建、资产闭包和发布检查。这样新增 API 的 MF +`pnpm release:version`，在候选中完成生产构建、独立 SDK、Job 观测、资产闭包和发布检查。这样新增 API 的 MF 可以明确要求下一兼容 SDK，而无需放宽为兼容旧已发布 SDK；原版本和 Changesets 保持不变。 +Pages preview 使用同一准备命令，从受信控制器所选的准确 PR head 建立派生候选。 +SPA、MF 与 Preview Registry facade 的生产者坐标都读取该候选；Python Toolkit 仍由原 +隔离 checkout 的冻结 PDM 环境执行。此流程不产生用于正式发布的版本或消费源分支的版本意图。 + 本地可运行 `node scripts/prepare-native-checkout.mjs --output .runtime/native-checkout --offline`， 随后进入输出的 `workspace` 执行 `pnpm build`。输出目录必须为空；准备过程记录原 HEAD、 是否有未提交变更与源快照摘要。它只为验证派生版本，不发布资产或创建提交。 diff --git a/extensions/bilibili/package.json b/extensions/bilibili/package.json index ba0e3859..f8db12ca 100644 --- a/extensions/bilibili/package.json +++ b/extensions/bilibili/package.json @@ -37,7 +37,7 @@ "nickname": "哔哩哔哩", "module_federation": { "host_sdk": "@inkcre/core", - "host_sdk_version": ">=0.4.4 <0.5.0" + "host_sdk_version": ">=0.4.5 <0.5.0" } }, "intent": { diff --git a/extensions/bilibili/tsconfig.json b/extensions/bilibili/tsconfig.json index 55268f80..e7b23556 100644 --- a/extensions/bilibili/tsconfig.json +++ b/extensions/bilibili/tsconfig.json @@ -8,7 +8,7 @@ "jsxImportSource": "vue", "skipLibCheck": true, "paths": { - "@/*": ["../../apps/client-web/src/*"], + "@/*": ["../../apps/web/src/*"], "@inkcre/core": ["../../packages/core/src/index.ts"], "@inkcre/core/*": ["../../packages/core/src/*"] }, diff --git a/extensions/bilibili/vite.config.ts b/extensions/bilibili/vite.config.ts index 55d52a4e..9a130db0 100644 --- a/extensions/bilibili/vite.config.ts +++ b/extensions/bilibili/vite.config.ts @@ -20,7 +20,7 @@ export const bilibiliArtifactBase = './' export const bilibiliBuildTarget = 'es2022' export const bilibiliBuildOptions = { target: bilibiliBuildTarget, - outDir: 'dist/client-web', + outDir: 'dist/web', sourcemap: true, } export const bilibiliFederationOptions = { diff --git a/scripts/tests/prepare-native-checkout.test.mjs b/scripts/tests/prepare-native-checkout.test.mjs index 63eb575b..3cfd24a5 100644 --- a/scripts/tests/prepare-native-checkout.test.mjs +++ b/scripts/tests/prepare-native-checkout.test.mjs @@ -11,13 +11,13 @@ async function fixture() { const output = join(root, 'candidate') const input = { 'package.json': { name: 'fixture', private: true }, - 'packages/core/package.json': { name: '@inkcre/core', version: '0.4.3' }, + 'packages/core/package.json': { name: '@inkcre/core', version: '0.4.4' }, 'extensions/example/package.json': { name: '@inkcre/example', version: '0.1.0', inkcre: { name: 'inkcre/example', - module_federation: { host_sdk: '@inkcre/core', host_sdk_version: '>=0.4.4 <0.5.0' }, + module_federation: { host_sdk: '@inkcre/core', host_sdk_version: '>=0.4.5 <0.5.0' }, }, }, '.changeset/compatible.md': '---\n"@inkcre/core": patch\n---\nCompatible capability\n', @@ -35,7 +35,7 @@ async function fixture() { if (command === 'pnpm' && args[0] === 'release:version') { await writeFile( join(cwd, 'packages/core/package.json'), - JSON.stringify({ name: '@inkcre/core', version: '0.4.4' }) + JSON.stringify({ name: '@inkcre/core', version: '0.4.5' }) ) await unlink(join(cwd, '.changeset/compatible.md')) } @@ -53,11 +53,11 @@ void test('the existing version command prepares an isolated candidate without c run: context.run, offline: true, }) - assert.equal(result.versions['@inkcre/core'].prepared, '0.4.4') + assert.equal(result.versions['@inkcre/core'].prepared, '0.4.5') assert.equal( JSON.parse(await readFile(join(context.source, 'packages/core/package.json'), 'utf8')) .version, - '0.4.3' + '0.4.4' ) assert.match( await readFile(join(context.source, '.changeset/compatible.md'), 'utf8'), @@ -115,7 +115,7 @@ void test('loosening a future producer range cannot masquerade as version prepar if (command === 'pnpm' && args[0] === 'release:version') { const file = join(cwd, 'extensions/example/package.json') const value = JSON.parse(await readFile(file, 'utf8')) - value.inkcre.module_federation.host_sdk_version = '>=0.4.3 <0.5.0' + value.inkcre.module_federation.host_sdk_version = '>=0.4.4 <0.5.0' await writeFile(file, JSON.stringify(value)) } return result diff --git a/scripts/verify-browser-automation.mjs b/scripts/verify-browser-automation.mjs index 6f33c042..30613500 100644 --- a/scripts/verify-browser-automation.mjs +++ b/scripts/verify-browser-automation.mjs @@ -10,7 +10,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url' import { chromium } from '@playwright/test' const root = fileURLToPath(new URL('../', import.meta.url)) -const require = createRequire(new URL('../apps/client-web/package.json', import.meta.url)) +const require = createRequire(new URL('../apps/web/package.json', import.meta.url)) const { build } = await import(pathToFileURL(require.resolve('vite')).href) await mkdir(join(root, '.runtime'), { recursive: true }) const temporary = await mkdtemp(join(root, '.runtime/browser-automation-')) @@ -129,7 +129,7 @@ try { emptyOutDir: true, }, }) - await cp(join(root, 'apps/client-webext/.output/chrome-mv3'), extension, { recursive: true }) + await cp(join(root, 'apps/webext/.output/chrome-mv3'), extension, { recursive: true }) const manifest = JSON.parse(await readFile(join(extension, 'manifest.json'), 'utf8')) // This isolated fixture pregrants only its synthetic host. Native permission UX is separate evidence. manifest.host_permissions = ['http://127.0.0.1/*'] From a26fcb4c8f8dffb71c2730a17074af663030abdf Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 00:17:56 +0800 Subject: [PATCH 03/12] =?UTF-8?q?fix(release):=20=E4=B8=BA=E5=B7=B2?= =?UTF-8?q?=E6=8F=90=E4=BA=A4=E7=9A=84=E7=89=88=E6=9C=AC=E6=84=8F=E5=9B=BE?= =?UTF-8?q?=E4=BF=9D=E7=95=99=E5=AE=8C=E6=95=B4=E5=80=99=E9=80=89=E5=8E=86?= =?UTF-8?q?=E5=8F=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 避免 Changesets 在无远端浅候选中反复查询提交来源 - Pages 使用完整准确 head 历史，并以真实 Git 和正式 CLI 验证已提交快照 --- .github/workflows/pages-preview.yml | 1 + .../native-extension-delivery.md | 2 + scripts/prepare-native-checkout.mjs | 17 ++-- .../tests/prepare-native-checkout.test.mjs | 91 +++++++++++++++++++ 4 files changed, 100 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pages-preview.yml b/.github/workflows/pages-preview.yml index e481f4e3..8ec0ab6c 100644 --- a/.github/workflows/pages-preview.yml +++ b/.github/workflows/pages-preview.yml @@ -72,6 +72,7 @@ jobs: with: ref: ${{ needs.identity.outputs.head_sha }} path: candidate + fetch-depth: 0 persist-credentials: false - name: Install the pinned pnpm release diff --git a/docs/40-deployment/native-extension-delivery.md b/docs/40-deployment/native-extension-delivery.md index 4bed95dd..edf6f971 100644 --- a/docs/40-deployment/native-extension-delivery.md +++ b/docs/40-deployment/native-extension-delivery.md @@ -79,6 +79,8 @@ Workspace contract 先在原 checkout 校验格式、静态规则和类型，再 Pages preview 使用同一准备命令，从受信控制器所选的准确 PR head 建立派生候选。 SPA、MF 与 Preview Registry facade 的生产者坐标都读取该候选；Python Toolkit 仍由原 隔离 checkout 的冻结 PDM 环境执行。此流程不产生用于正式发布的版本或消费源分支的版本意图。 +准备入口要求完整的所选源码历史，隔离候选保留该历史供 Changesets 查询变更记录； +CI 与 Pages checkout 均使用 `fetch-depth: 0`，不依赖运行中尝试加深一个没有远端的浅候选。 本地可运行 `node scripts/prepare-native-checkout.mjs --output .runtime/native-checkout --offline`， 随后进入输出的 `workspace` 执行 `pnpm build`。输出目录必须为空；准备过程记录原 HEAD、 diff --git a/scripts/prepare-native-checkout.mjs b/scripts/prepare-native-checkout.mjs index 43d81f1f..c25d98a3 100644 --- a/scripts/prepare-native-checkout.mjs +++ b/scripts/prepare-native-checkout.mjs @@ -75,6 +75,11 @@ export async function prepareNativeCheckout({ run = execute, }) { const source = await realpath(sourceDirectory) + const shallow = String(await run('git', ['rev-parse', '--is-shallow-repository'], source)).trim() + if (shallow === 'true') + throw new Error( + 'Version preparation requires complete source history; fetch the selected checkout with fetch-depth: 0' + ) const output = resolve(outputDirectory) if (output === source || source.startsWith(output + sep)) throw new Error('Candidate cannot replace the source or its ancestor') @@ -108,17 +113,7 @@ export async function prepareNativeCheckout({ /** @type {Array<[string, string[]]>} */ const steps = [ ['git', ['init', '--quiet']], - [ - 'git', - [ - 'fetch', - '--quiet', - '--no-tags', - '--depth=1', - source, - `${revision}:refs/remotes/origin/main`, - ], - ], + ['git', ['fetch', '--quiet', '--no-tags', source, `${revision}:refs/remotes/origin/main`]], ['git', ['update-ref', 'refs/heads/main', revision]], ['git', ['symbolic-ref', 'HEAD', 'refs/heads/main']], ['git', ['read-tree', revision]], diff --git a/scripts/tests/prepare-native-checkout.test.mjs b/scripts/tests/prepare-native-checkout.test.mjs index 3cfd24a5..0239f151 100644 --- a/scripts/tests/prepare-native-checkout.test.mjs +++ b/scripts/tests/prepare-native-checkout.test.mjs @@ -1,10 +1,14 @@ import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' import { mkdtemp, mkdir, readFile, rm, writeFile, unlink } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { fileURLToPath } from 'node:url' import test from 'node:test' import { prepareNativeCheckout } from '../prepare-native-checkout.mjs' +const repository = fileURLToPath(new URL('../..', import.meta.url)) + async function fixture() { const root = await mkdtemp(join(tmpdir(), 'inkcre-native-checkout-')) const source = join(root, 'source') @@ -29,6 +33,7 @@ async function fixture() { const calls = [] const run = async (command, args, cwd) => { calls.push({ command, args, cwd }) + if (command === 'git' && args.includes('--is-shallow-repository')) return 'false' if (command === 'git' && args[0] === 'rev-parse') return 'a'.repeat(40) if (command === 'git' && args[0] === 'status') return ' M packages/core/package.json\n' if (command === 'git' && args[0] === 'ls-files') return Object.keys(input).join('\0') + '\0' @@ -84,6 +89,92 @@ void test('the existing version command prepares an isolated candidate without c } }) +void test('committed version intent is resolved by the actual CLI from complete isolated Git history', async () => { + const context = await fixture() + const run = async (command, args, cwd) => { + if (command === 'pnpm' && args[0] === 'install') return '' + return execFileSync( + command === 'pnpm' ? join(repository, 'node_modules/.bin/changeset') : command, + command === 'pnpm' ? ['version'] : args, + { cwd, encoding: 'utf8', timeout: 15_000 } + ) + } + try { + await writeFile( + join(context.source, 'pnpm-workspace.yaml'), + 'packages:\n - packages/*\n - extensions/*\n' + ) + await writeFile( + join(context.source, '.changeset/config.json'), + JSON.stringify({ + changelog: '@changesets/cli/changelog', + commit: false, + fixed: [], + linked: [], + access: 'restricted', + baseBranch: 'main', + updateInternalDependencies: 'patch', + ignore: [], + }) + ) + execFileSync('git', ['init', '--quiet', '-b', 'main'], { cwd: context.source }) + execFileSync('git', ['add', '.'], { cwd: context.source }) + execFileSync( + 'git', + [ + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.test', + 'commit', + '--quiet', + '-m', + 'Initial native packages', + ], + { cwd: context.source } + ) + await writeFile( + join(context.source, 'README.md'), + 'Version intent is committed before candidate preparation.\n' + ) + execFileSync('git', ['add', 'README.md'], { cwd: context.source }) + execFileSync( + 'git', + [ + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.test', + 'commit', + '--quiet', + '-m', + 'Prepare source checkpoint', + ], + { cwd: context.source } + ) + const result = await prepareNativeCheckout({ + sourceDirectory: context.source, + outputDirectory: context.output, + run, + }) + assert.equal(result.source_modified, false) + assert.equal(result.versions['@inkcre/core'].prepared, '0.4.5') + assert.equal( + execFileSync('git', ['rev-parse', '--is-shallow-repository'], { + cwd: result.candidate, + encoding: 'utf8', + }).trim(), + 'false' + ) + assert.match( + await readFile(join(context.source, '.changeset/compatible.md'), 'utf8'), + /Compatible capability/ + ) + } finally { + await rm(context.root, { recursive: true, force: true }) + } +}) + void test('a version command that leaves the Host below the producer minimum is rejected', async () => { const context = await fixture() try { From 42ae75bd2c008ae7d27a25fa15b46f87242e7efe Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 00:21:53 +0800 Subject: [PATCH 04/12] =?UTF-8?q?docs(shared):=20=E5=BC=95=E5=85=A5=20Sour?= =?UTF-8?q?ce=20=E6=89=B9=E6=AC=A1=E4=B8=8E=20B=20=E7=AB=99=E6=94=B6?= =?UTF-8?q?=E8=97=8F=E5=90=88=E5=90=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/_shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_shared b/docs/_shared index 17fc19e7..76dd7c65 160000 --- a/docs/_shared +++ b/docs/_shared @@ -1 +1 @@ -Subproject commit 17fc19e7230187a7a989815e7b8303bbb380f478 +Subproject commit 76dd7c655ba7dcc1aeb295dac08ada194f8d662a From 86fe694660a3d3f9b737caab5ec6afa9d199b04e Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 02:08:11 +0800 Subject: [PATCH 05/12] =?UTF-8?q?docs(shared):=20=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E6=9C=80=E5=B0=8F=E6=94=B6=E8=97=8F=E6=8C=81=E4=B9=85=E5=8C=96?= =?UTF-8?q?=E5=A5=91=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/_shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_shared b/docs/_shared index 76dd7c65..b50921b1 160000 --- a/docs/_shared +++ b/docs/_shared @@ -1 +1 @@ -Subproject commit 76dd7c655ba7dcc1aeb295dac08ada194f8d662a +Subproject commit b50921b19cb6cf81ad4c09786423195974d901b2 From 761bfa9c43c71de589d3f2b47f887b46507d88b9 Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 02:17:53 +0800 Subject: [PATCH 06/12] =?UTF-8?q?fix(bilibili):=20=E7=94=A8=E6=99=AE?= =?UTF-8?q?=E9=80=9A=E4=BF=A1=E6=81=AF=E5=9B=BE=E5=AE=8C=E6=88=90=E6=9C=80?= =?UTF-8?q?=E5=B0=8F=E9=87=87=E9=9B=86=E6=8C=81=E4=B9=85=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 扩展按原生身份和成员关系复用图，写完本页后再保存进度 - 移除新数据库协议、额外类型生成入口与候选准备框架 - 验证跨 Peer 复用、复采、部分写失败及未知结果恢复 --- .github/workflows/ci.yml | 13 +- .github/workflows/pages-preview.yml | 21 +- docs/30-unit-tdd/bilibili-extension.md | 10 +- docs/40-deployment/development-runtime.md | 21 +- .../native-extension-delivery.md | 20 +- extensions/bilibili/README.md | 4 +- extensions/bilibili/src/collect.ts | 2 +- extensions/bilibili/src/repository.ts | 255 ++++++++++-- extensions/bilibili/src/source.ts | 5 +- extensions/bilibili/tests/favorites.test.ts | 4 +- extensions/bilibili/tests/repository.test.ts | 375 ++++++++++++++++++ .../core/src/database/database.generated.ts | 39 -- .../database/runtime-contract.generated.json | 2 +- scripts/check-local-runtime-contract.mjs | 18 +- scripts/database-contract-source.mjs | 16 - scripts/database-provider.mjs | 6 +- scripts/doctor.mjs | 2 +- scripts/prepare-native-checkout.mjs | 185 --------- scripts/sync-database-contract.mjs | 65 +-- .../tests/database-contract-source.test.mjs | 38 -- .../tests/prepare-native-checkout.test.mjs | 278 ------------- svc.json | 190 +++------ 22 files changed, 731 insertions(+), 838 deletions(-) create mode 100644 extensions/bilibili/tests/repository.test.ts delete mode 100644 scripts/database-contract-source.mjs delete mode 100644 scripts/prepare-native-checkout.mjs delete mode 100644 scripts/tests/database-contract-source.test.mjs delete mode 100644 scripts/tests/prepare-native-checkout.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 170a1430..d5eb306b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,23 +113,22 @@ jobs: - name: 校验源代码格式、静态规则和类型 run: pnpm format:check && pnpm lint && pnpm type-check - - name: 以正式版本命令准备隔离验证坐标 - run: node scripts/prepare-native-checkout.mjs --output "$RUNNER_TEMP/native-checkout" + - name: 在临时 checkout 中准备兼容构建版本 + run: | + git show-ref --verify --quiet refs/heads/main || git branch main HEAD + pnpm release:version + pnpm install --lockfile-only --ignore-scripts - name: 构建并校验相互兼容的 Host 与 MF 产物 - working-directory: ${{ runner.temp }}/native-checkout/workspace run: pnpm build - name: Verify independent SDK and Job observations - working-directory: ${{ runner.temp }}/native-checkout/workspace run: pnpm check:sdk && pnpm check:web - name: 验证 B 站收藏采集规则 - working-directory: ${{ runner.temp }}/native-checkout/workspace run: pnpm --filter @inkcre/ext-bilibili test - name: Verify independent MF publication - working-directory: ${{ runner.temp }}/native-checkout/workspace run: >- node --test scripts/test-extension-publication.mjs scripts/test-core-sdk-publication.mjs scripts/tests/independent-extension-release.test.mjs @@ -139,8 +138,6 @@ jobs: scripts/tests/browser-automation-transport.test.mjs scripts/tests/job-cancellation.test.mjs scripts/tests/source-anchor.test.mjs - scripts/tests/prepare-native-checkout.test.mjs - scripts/tests/database-contract-source.test.mjs peer-database: # Existing branch protection binds this exact check name. diff --git a/.github/workflows/pages-preview.yml b/.github/workflows/pages-preview.yml index 8ec0ab6c..d09ab775 100644 --- a/.github/workflows/pages-preview.yml +++ b/.github/workflows/pages-preview.yml @@ -103,14 +103,15 @@ jobs: env: PDM_CHECK_UPDATE: 'false' - - name: Prepare isolated preview versions with the release command - run: >- - node candidate/scripts/prepare-native-checkout.mjs - --source candidate - --output "$RUNNER_TEMP/pages-native-checkout" + - name: Prepare preview versions in the temporary checkout + working-directory: candidate + run: | + git show-ref --verify --quiet refs/heads/main || git branch main HEAD + pnpm release:version + pnpm install --lockfile-only --ignore-scripts - name: Build the exact-head SPA and Module Federation snapshots - working-directory: ${{ runner.temp }}/pages-native-checkout/workspace + working-directory: candidate run: pnpm build - name: Build the same-origin Preview Registry facade @@ -119,11 +120,11 @@ jobs: PREVIEW_ORIGIN: https://preview-client-web-pr-${{ needs.identity.outputs.pull_number }}.${{ vars.CLOUDFLARE_PAGES_PROJECT }}.pages.dev run: | pdm run inkcre-ext preview build \ - --inventory "$RUNNER_TEMP/pages-native-checkout/workspace/.github/preview/extensions.json" \ + --inventory .github/preview/extensions.json \ --public-origin "$PREVIEW_ORIGIN" \ - --output "$RUNNER_TEMP/pages-native-checkout/workspace/.preview-registry" - cp -R "$RUNNER_TEMP/pages-native-checkout/workspace/.preview-registry/." "$RUNNER_TEMP/pages-native-checkout/workspace/apps/web/dist/" - cp -R "$RUNNER_TEMP/pages-native-checkout/workspace/apps/web/dist" "$GITHUB_WORKSPACE/.pages-preview" + --output .preview-registry + cp -R .preview-registry/. apps/web/dist/ + cp -R apps/web/dist "$GITHUB_WORKSPACE/.pages-preview" - name: Bind Module Federation assets to the selected snapshot env: diff --git a/docs/30-unit-tdd/bilibili-extension.md b/docs/30-unit-tdd/bilibili-extension.md index 2a351fe7..9c5680af 100644 --- a/docs/30-unit-tdd/bilibili-extension.md +++ b/docs/30-unit-tdd/bilibili-extension.md @@ -12,9 +12,11 @@ 四个精确 Resolver 为 `extensions.bilibili.user.v1`、`folder.v1`、`video.v1`、`membership.v1`，完整前缀相同。领域内容校验由 `schema.ts` 执行。账号以 MID、收藏夹以完整 media_id、视频以 aid 标识；BVID 是已知视频别名，原始 fid 不能代替 media_id。 -数据库 identity namespace 为 `inkcre/bilibili`，key 分别为 `user:{mid}`、`folder:{media_id}`、`video:{aid}`、`membership:{media_id}:{aid}`。账号到收藏夹使用 `owns`，收藏夹到成员使用 `contains`，成员到视频使用 `item`，视频到 UP 主使用 `author`。成员自己的 Block 承载可变收藏时间、观察位置与条目可用性；稳定边不把这些值串入身份。观察位置明确为 `page_number` 与 `position_in_page`，不由分页大小推算并声称原生绝对顺序。内容不重复保存跨实体关联作为第二份权威。 +账号到收藏夹使用 `owns`，收藏夹到成员使用 `contains`，成员到视频使用 `item`，视频到 UP 主使用 `author`。成员自己的 Block 承载可变收藏时间、观察位置与条目可用性；稳定边不把这些值串入身份。观察位置明确为 `page_number` 与 `position_in_page`，不由分页大小推算并声称原生绝对顺序。内容不重复保存跨实体关联作为第二份权威。 -来源没有取得的字段不写入 patch，数据库顶层合并保留已有事实。失效视频的占位标题、空缺作者或简介不能抹掉已保存事实。独立视频身份不依赖某次 Source 或浏览器，重复采集和跨收藏夹采集复用相同 Block。 +扩展通过普通 Block / Relation API 自行核对原生对象，每次执行读取已有 B 站内联 JSON 块，以实际解析出的 MID、完整 media_id 或 aid 复用 Block。查询按 Resolver 和普通分页限定范围，JSON 空格与属性顺序不参与身份判断。成员身份来自明确的收藏夹 `contains` 成员 `item` 视频路径；v1 成员内容不增加 folder_id、aid 或私有 native_key。执行中的内存查找表不作为共享权威，也不保存第二份持久身份账本。 + +来源没有取得的字段不写入 patch，扩展在已有 JSON 对象上合并已知字段。失效视频的占位标题、空缺作者或简介不能抹掉已保存事实。独立视频身份不依赖某次 Source 或浏览器，重复采集和跨收藏夹采集复用相同 Block；本次采集只补齐自己的稳定边，不删除用户关系。 ## 批次与恢复 @@ -24,7 +26,9 @@ Source.state 仅保存范围、分页提示、首批和已提交边界的身份 末批提交前重新读取首批，对照本轮首批身份、起始报告数量与末批报告数量；发生变化时不提交“完成”，保留此前已确认的批次，并使 Job 明确失败。完整遍历与条目全数取得是不同声明：每个收藏夹同时保留累计返回、解析视频、失效、未支持、未解析数量和来源报告数，设置界面展示差额。数量不等本身不会触发无限重扫，也不能被隐去后声称完整；此边界检查不检测任意中间位置变化。 -每批使用准入数据库 RPC `commit_source_batch`，同时写入原生身份映射、图、Source 锚点与下一进度。Source 配置和状态以 CAS 检查，避免并发采集覆盖进度。客户端只在确认提交后采用下一进度；失败和未知结果不自动重放。`Source.block` 指向该 Source 独占的 `core.source.v1` 锚点，锚点以稳定 `collects` 边指向共享账号业务根。RPC 的 `root_key` 选择业务根，回包 `source_block` 才是可赋给 Source.block 的锚点 ID；不同 Source 可采集同一个账号而保持各自来源身份。设置变更只更新配置，不顺便写业务图。 +每页先确认对象和关系写入，最后以普通 Source 更新保存进度。失败停止，创建结果未知时不在同一次执行中重放。下一次执行重新读取普通图：已可确认身份的对象和完整成员路径会被复用，缺失的边会被补齐；在成员身份路径尚未形成前中断可能留下孤立块，不为消除此残留增加持久账本。本版不保证并发首次创建唯一或整页原子可见。 + +`Source.block` 指向该 Source 独占的 `core.source.v1` 锚点，锚点以稳定 `collects` 边指向共享账号业务根。没有锚点引用时先按普通图中的 Source id 查找，避免因一次来源锚点创建响应丢失就直接重复创建；锚点和业务账号不能互换。最后更新 Source 的锚点引用与进度。不同 Source 可采集同一个账号而保持各自来源身份。设置变更只更新配置，不顺便写业务图。 ## 读取 diff --git a/docs/40-deployment/development-runtime.md b/docs/40-deployment/development-runtime.md index 74fa0de8..74618660 100644 --- a/docs/40-deployment/development-runtime.md +++ b/docs/40-deployment/development-runtime.md @@ -22,14 +22,13 @@ The committed [`svc.json`](../../svc.json) declares four worktree-scoped capabilities: `database`, `web`, `web-ui`, and `webext`. SVC resolves the worktree instance and coordinates provisioning and -readiness through schema 5 resources, actions and observations; [`scripts/dev.mjs`](../../scripts/dev.mjs) and the database runtime scripts own the +readiness; [`scripts/dev.mjs`](../../scripts/dev.mjs) and the database runtime scripts own the executable behavior. Observe without provisioning with `pnpm dev:status`, and run `pnpm run doctor` before diagnosing host setup. Portless gives the web and WXT targets instance-specific HTTPS `.localhost` routes. Their probes -accept only the expected target and worktree identity. Database provisioning completes on exit and -the ready observation; long-running web and WXT actions complete on readiness and retain their -managed process owner. The optional `web-ui` target is a non-release +accept only the expected target and worktree identity. Provisioning returns after readiness; it does +not transfer long-lived process ownership to SVC. The optional `web-ui` target is a non-release source lane whose resolved checkout stays machine-local. Normal build, check, and CI continue to use the locked Registry package. @@ -154,20 +153,6 @@ Pull-request evidence does not establish source-branch or migration equality bet 独占的 typegen 数据库，不接受可移动 tag，也不会拉取本地 ID。此入口不支持 external provider；需选择构建候选的 同一 provider。`contracts/core-release.json` 仍只接纳已发布的 GHCR digest，本地生成不能代替发布产物验收。 -联合开发尚未发布的数据库协议时，也可以从已就绪的 Python 工作树 runtime 生成类型，不拉取稳定镜像或创建第二个数据库： - -```bash -# 在拥有该 runtime 的 impl-py 工作树导出当前协议。 -pdm run db:contract > .runtime/current-contract.json - -# 在 impl-ts 工作树显式指定 owner descriptor 与刚导出的合同。 -pnpm contract:sync --runtime /absolute/impl-py/.runtime/database/INSTANCE/runtime.json --runtime-contract /absolute/impl-py/.runtime/current-contract.json -``` - -此入口只接受 `InKCre/impl-py` 的 development descriptor，合同 revision 与 migration head 必须匹配。 -它通过 descriptor 指定的数据库 owner 运行 Supabase CLI，读取本机 credential 文件但不输出连接凭据；不停止或修改 owner runtime。 -追加 `--check` 验证生成结果。未传这两个参数时仍使用已发布镜像路径；本地生成不会修改 canonical candidate，也不能作为正式发布证据。 - ## Operator Commands Run these from the repository root; [`package.json`](../../package.json) owns their exact carriers. diff --git a/docs/40-deployment/native-extension-delivery.md b/docs/40-deployment/native-extension-delivery.md index edf6f971..bb158cc5 100644 --- a/docs/40-deployment/native-extension-delivery.md +++ b/docs/40-deployment/native-extension-delivery.md @@ -71,20 +71,12 @@ copy so the preview can load a mutually compatible Host and MF when a change rai Host SDK. This produces temporary derived versions for preview only. It neither consumes the pending changesets in the PR nor replaces the normal Version PR and protected-main release lifecycle. -Workspace contract 先在原 checkout 校验格式、静态规则和类型，再通过 -`scripts/prepare-native-checkout.mjs` 将当前未忽略的源文件复制到隔离候选，运行正式 -`pnpm release:version`，在候选中完成生产构建、独立 SDK、Job 观测、资产闭包和发布检查。这样新增 API 的 MF -可以明确要求下一兼容 SDK，而无需放宽为兼容旧已发布 SDK；原版本和 Changesets 保持不变。 - -Pages preview 使用同一准备命令，从受信控制器所选的准确 PR head 建立派生候选。 -SPA、MF 与 Preview Registry facade 的生产者坐标都读取该候选；Python Toolkit 仍由原 -隔离 checkout 的冻结 PDM 环境执行。此流程不产生用于正式发布的版本或消费源分支的版本意图。 -准备入口要求完整的所选源码历史，隔离候选保留该历史供 Changesets 查询变更记录； -CI 与 Pages checkout 均使用 `fetch-depth: 0`，不依赖运行中尝试加深一个没有远端的浅候选。 - -本地可运行 `node scripts/prepare-native-checkout.mjs --output .runtime/native-checkout --offline`， -随后进入输出的 `workspace` 执行 `pnpm build`。输出目录必须为空；准备过程记录原 HEAD、 -是否有未提交变更与源快照摘要。它只为验证派生版本，不发布资产或创建提交。 +Workspace contract 先校验源代码格式、静态规则和类型，再直接在 CI 的临时 checkout 中运行 +`pnpm release:version` 与 `pnpm install --lockfile-only --ignore-scripts`，然后构建并验证 Host 和 MF。 +Pages preview 在受信控制器选定的 PR head 临时 checkout 中使用同样的正式版本命令，保留完整历史供 Changesets 读取。 +这些派生版本仅用于验证和预览，不回写源分支，也不替代 Version PR 或正式发布。 + +本地需要验证尚未发布的 SDK 时，在临时 clone 中运行上述命令及 `pnpm build`；日常工作树继续保留版本意图。 A plugin owns compatibility of its shared configuration, state, Source/Job parameters and stored information. Changesets record MF changes even when no Python change is necessary. Published native diff --git a/extensions/bilibili/README.md b/extensions/bilibili/README.md index 9af1bce5..a068f21f 100644 --- a/extensions/bilibili/README.md +++ b/extensions/bilibili/README.md @@ -27,7 +27,9 @@ MF Distribution 提供采集与独立读取能力。Python 的只读 Distributio “分页已遍历”仅表示可见分页到达结束标记并通过上述边界检查。界面同时显示累计返回条目、来源报告数量和差额；来源可能隐藏部分条目或异步更新计数，差额不能被当作已成功取得。列表在采集期间变化时，本版不承诺某一时刻的原子快照，也不声称能检测任意中间位置的重排。 -每批图变化和进度通过数据库 `commit_source_batch` 同事务确认。未知写入结果不会自动重放。再次采集复用原生身份和既有 Block，保留用户添加的关系；本版不会根据未出现的条目删除历史收藏资料。 +采集扩展通过普通 Block / Relation API 读取并核对已有的原生对象，复用 Block ID、合并已取得的字段，并保留用户添加的关系。每页图写入全部确认后，才更新 Source 进度；中途失败会停止，已经写入的部分仍可能可见，下一次执行重新读取普通图并补齐。本版不承诺整页原子可见或并发首次创建唯一，也不会根据未出现的条目删除历史收藏资料。 + +创建结果未知时不会自动重放。若收藏成员在关联尚未连齐时中断，可能留下无法确认原生身份的孤立块；下一次只按明确的收藏夹—成员—视频路径复用成员，不猜测或误认其他条目。 失效占位标题不会覆盖已有标题或简介。非视频和身份不足条目计入未支持或未解析数量。未登录、账号变化、挑战响应、分页异常或断连会使任务明确失败，不能被当成空收藏夹成功。 diff --git a/extensions/bilibili/src/collect.ts b/extensions/bilibili/src/collect.ts index 5dcf773f..9fcb1dd2 100644 --- a/extensions/bilibili/src/collect.ts +++ b/extensions/bilibili/src/collect.ts @@ -53,7 +53,7 @@ export async function collectFavorites( } const next = advanceProgress(progress, folder, current, cursor.next_page, options.max_pages) await commit(account, current, next) - // Progress is adopted only after the database confirms the graph and cursor together. + // Persisted progress follows the page's confirmed graph writes. progress = next cursor = progress.folders[folder]! } diff --git a/extensions/bilibili/src/repository.ts b/extensions/bilibili/src/repository.ts index 498f0200..783af44f 100644 --- a/extensions/bilibili/src/repository.ts +++ b/extensions/bilibili/src/repository.ts @@ -1,37 +1,232 @@ -import { Source, type Json } from '@inkcre/core' -import { z } from 'zod' -import { extensionName } from './schema.ts' -import type { SourceGraphBatch } from './graph.ts' +import { Block, Relation, Source } from '@inkcre/core' +import { NativeIdSchema, resolverTypes } from './schema.ts' +import type { GraphRecord, SourceGraphBatch } from './graph.ts' import type { Progress } from './progress.ts' -const committedBatch = z.object({ - blocks: z.record(z.string(), z.number().int().positive()), - state: z.record(z.string(), z.unknown()), - source_block: z.number().int().positive().nullable(), -}) +type StoredBlock = Pick -function json(value: unknown): Json { - const encoded = JSON.stringify(value) - if (encoded === undefined) throw new TypeError('采集批次需要可序列化的 JSON。') - return z.json().parse(JSON.parse(encoded)) +function objectContent(content: string): Record | null { + try { + const value: unknown = JSON.parse(content) + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : null + } catch { + return null + } } -export async function commitBatch(source: Source, graph: SourceGraphBatch, progress: Progress) { - const result = await Source.dbApi - .rpc('commit_source_batch', { - source_id: source.id, - namespace: extensionName, - expected_config: json(source.config), - expected_state: json(source.state), - root_key: `user:${progress.scope.account_mid}`, - next_state: json({ ...source.state, bilibili: progress }), - records: json(graph.records), - relations: json(graph.relations), +function nativeKey(resolver: string, content: Record): string | null { + const kind = + resolver === resolverTypes.user + ? 'user' + : resolver === resolverTypes.folder + ? 'folder' + : resolver === resolverTypes.video + ? 'video' + : null + if (!kind) return null + const identity = NativeIdSchema.safeParse( + content[kind === 'user' ? 'mid' : kind === 'folder' ? 'id' : 'aid'] + ) + return identity.success ? `${kind}:${identity.data}` : null +} + +/** Bilibili reconciles its own native objects over the ordinary shared graph. */ +export class BilibiliRepository { + private loaded = false + private readonly records = new Map() + private readonly blocks = new Map() + private readonly memberships = new Map>() + + private async load() { + if (this.loaded) return + let offset = 0 + while (true) { + const result = await Block.dbApi + .from() + .select('*', { count: 'exact' }) + .in('resolver', Object.values(resolverTypes)) + .is('storage', null) + .order('id', { ascending: true }) + .range(offset, offset + 999) + .throwOnError() + const rows = result.data ?? [] + for (const row of rows) { + this.blocks.set(row.id, row) + const content = objectContent(row.content) + const key = content && nativeKey(row.resolver, content) + if (key && !this.records.has(key)) this.records.set(key, row) + } + offset += rows.length + if (!rows.length || (result.count !== null ? offset >= result.count : rows.length < 1000)) + break + } + this.loaded = true + } + + private async saveRecord(record: GraphRecord): Promise { + const key = nativeKey(record.resolver, record.content) + if (!key) throw new Error('B 站对象缺少可确认的原生身份。') + const previous = this.records.get(key) + const saved = await this.saveContent(previous, record.resolver, record.content) + this.records.set(key, saved) + this.blocks.set(saved.id, saved) + return saved + } + + private async saveContent( + previous: StoredBlock | undefined, + resolver: string, + patch: Record + ) { + const original = previous ? objectContent(previous.content) : null + const value = { ...original, ...patch } + if (previous && JSON.stringify(original) === JSON.stringify(value)) return previous + const content = JSON.stringify(value) + // A lost create response ends this attempt. A later execution reads ordinary Blocks again. + const query = previous + ? Block.dbApi.update({ content }).eq('id', previous.id) + : Block.dbApi.insert({ resolver, storage: null, content }) + const result = await query.select().single().throwOnError() + return result.data + } + + private async folderMemberships(folder: number): Promise> { + const cached = this.memberships.get(folder) + if (cached) return cached + const found = new Map() + let offset = 0 + while (true) { + const result = await Relation.dbApi + .from() + .select('*', { count: 'exact' }) + .eq('from_', folder) + .eq('content', 'contains') + .order('id', { ascending: true }) + .range(offset, offset + 999) + .throwOnError() + const rows = result.data ?? [] + const ids = rows.flatMap((row) => (row.to_ === null ? [] : [row.to_])) + for (let index = 0; index < ids.length; index += 200) { + const items = await Relation.dbApi + .from() + .select() + .in('from_', ids.slice(index, index + 200)) + .eq('content', 'item') + .throwOnError() + for (const memberId of ids.slice(index, index + 200)) { + const membership = this.blocks.get(memberId) + const links = (items.data ?? []).filter((row) => row.from_ === memberId) + if (membership?.resolver !== resolverTypes.membership || links.length !== 1) continue + const video = links[0]!.to_ === null ? undefined : this.blocks.get(links[0]!.to_) + const content = video && objectContent(video.content) + const key = video && content && nativeKey(video.resolver, content) + if (!video || !key || this.records.get(key)?.id !== video.id) continue + if (!found.has(video.id)) found.set(video.id, membership) + } + } + offset += rows.length + if (!rows.length || (result.count !== null ? offset >= result.count : rows.length < 1000)) + break + } + this.memberships.set(folder, found) + return found + } + + private async link(from: number, to: number, content: string) { + const existing = await Relation.dbApi + .from() + .select('id') + .eq('from_', from) + .eq('to_', to) + .eq('content', content) + .limit(1) + .throwOnError() + if (existing.data?.length) return + await Relation.dbApi.insert({ from_: from, to_: to, content }).throwOnError() + } + + private async anchor(source: Source): Promise { + let anchor: StoredBlock | undefined + if (source.block !== null) { + const result = await Block.dbApi + .from() + .select() + .eq('id', source.block) + .single() + .throwOnError() + anchor = result.data + } else { + const result = await Block.dbApi + .from() + .select() + .eq('resolver', 'core.source.v1') + .order('id', { ascending: true }) + .throwOnError() + anchor = result.data?.find((row) => objectContent(row.content)?.id === source.id) + } + if ( + anchor && + (anchor.resolver !== 'core.source.v1' || objectContent(anchor.content)?.id !== source.id) + ) + throw new Error('采集源的来源锚点不匹配，已停止写入。') + return this.saveContent(anchor, 'core.source.v1', { + id: source.id, + type: source.type, + nickname: source.nickname, }) - .throwOnError() - // An ambiguous transport failure is deliberately not retried here. - const committed = committedBatch.parse(result.data) - source.state = committed.state - source.block = committed.source_block - return committed + } + + async reconcilePage(source: Source, graph: SourceGraphBatch, progress: Progress): Promise { + await this.load() + const ids = new Map() + for (const record of graph.records) { + if (record.resolver === resolverTypes.membership) continue + ids.set(record.key, (await this.saveRecord(record)).id) + } + for (const record of graph.records) { + if (record.resolver !== resolverTypes.membership) continue + const folderKey = graph.relations.find( + (edge) => edge.content === 'contains' && edge.to_key === record.key + )?.from_key + const videoKey = graph.relations.find( + (edge) => edge.content === 'item' && edge.from_key === record.key + )?.to_key + const folder = folderKey && ids.get(folderKey) + const video = videoKey && ids.get(videoKey) + if (!folder || !video) throw new Error('收藏条目缺少明确的收藏夹或视频关联。') + const memberships = await this.folderMemberships(folder) + const membership = await this.saveContent( + memberships.get(video), + record.resolver, + record.content + ) + await this.link(membership.id, video, 'item') + await this.link(folder, membership.id, 'contains') + memberships.set(video, membership) + this.blocks.set(membership.id, membership) + ids.set(record.key, membership.id) + } + for (const edge of graph.relations) { + if (edge.content === 'contains' || edge.content === 'item') continue + const from = ids.get(edge.from_key) + const to = ids.get(edge.to_key) + if (!from || !to) throw new Error('B 站资料关联端点缺失。') + await this.link(from, to, edge.content) + } + const root = ids.get(`user:${progress.scope.account_mid}`) + if (!root) throw new Error('本页缺少已确认的 B 站账号。') + const anchor = await this.anchor(source) + await this.link(anchor.id, root, 'collects') + const state = { ...source.state, bilibili: progress } + await Source.dbApi + .update({ block: anchor.id, state }) + .eq('id', source.id) + .select('id') + .single() + .throwOnError() + source.block = anchor.id + source.state = state + } } diff --git a/extensions/bilibili/src/source.ts b/extensions/bilibili/src/source.ts index f77bdec0..ca5a3e5b 100644 --- a/extensions/bilibili/src/source.ts +++ b/extensions/bilibili/src/source.ts @@ -2,7 +2,7 @@ import { BrowserAccess, type SourceImplementation } from '@inkcre/core' import { CollectConfigSchema, SourceConfigSchema } from './schema.ts' import { favoriteGraph } from './graph.ts' import { startProgress } from './progress.ts' -import { commitBatch } from './repository.ts' +import { BilibiliRepository } from './repository.ts' import { FavoriteSession } from './site.ts' import { collectFavorites } from './collect.ts' @@ -15,6 +15,7 @@ export const bilibiliSource: SourceImplementation { @@ -25,7 +26,7 @@ export const bilibiliSource: SourceImplementation { - await commitBatch(source, favoriteGraph(account, current), next) + await repository.reconcilePage(source, favoriteGraph(account, current), next) } ) } diff --git a/extensions/bilibili/tests/favorites.test.ts b/extensions/bilibili/tests/favorites.test.ts index a6b4cd72..fbd777dc 100644 --- a/extensions/bilibili/tests/favorites.test.ts +++ b/extensions/bilibili/tests/favorites.test.ts @@ -215,10 +215,10 @@ void test('a changed head restarts scanning and failed persistence never advance new AbortController().signal, session(changed, requested), async () => { - throw new Error('transaction rejected') + throw new Error('page write failed') } ), - /transaction rejected/ + /page write failed/ ) assert.deepEqual(requested, [1]) assert.deepEqual(persisted, before) diff --git a/extensions/bilibili/tests/repository.test.ts b/extensions/bilibili/tests/repository.test.ts new file mode 100644 index 00000000..f171de69 --- /dev/null +++ b/extensions/bilibili/tests/repository.test.ts @@ -0,0 +1,375 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { Block, Relation, Source } from '@inkcre/core' +import { BilibiliRepository } from '../src/repository.ts' +import { favoriteGraph } from '../src/graph.ts' +import { advanceProgress, startProgress } from '../src/progress.ts' +import { resolverTypes, SourceConfigSchema, sourceType } from '../src/schema.ts' +import type { FavoritePage } from '../src/normalize.ts' + +type Table = 'blocks' | 'relations' | 'sources' +type Row = { id: number; [key: string]: unknown } +type Values = Record +class Query { + table: Table + database: Database + action: 'read' | 'insert' | 'update' = 'read' + values: Values = {} + filters: ((row: Row) => boolean)[] = [] + start = 0 + end = Infinity + one = false + counted = false + constructor(database: Database, table: Table) { + this.database = database + this.table = table + } + select(_fields = '*', options?: { count: string }) { + this.counted = options?.count === 'exact' + return this + } + eq(key: string, value: unknown) { + this.filters.push((row) => row[key] === value) + return this + } + is(key: string, value: unknown) { + return this.eq(key, value) + } + in(key: string, values: unknown[]) { + this.filters.push((row) => values.includes(row[key])) + return this + } + order() { + return this + } + range(start: number, end: number) { + this.start = start + this.end = end + return this + } + limit(limit: number) { + this.end = this.start + limit - 1 + return this + } + single() { + this.one = true + return this + } + async throwOnError() { + return this.database.execute(this) + } +} +class Database { + rows: Record = { blocks: [], relations: [], sources: [] } + events: { table: Table; action: string; values: Values }[] = [] + failBefore: ((query: Query) => boolean) | null = null + failAfter: ((query: Query) => boolean) | null = null + pageLimit = 500 + client(table: Table) { + return { + from: () => new Query(this, table), + insert: (values: Values) => { + const query = new Query(this, table) + query.action = 'insert' + query.values = values + return query + }, + update: (values: Values) => { + const query = new Query(this, table) + query.action = 'update' + query.values = values + return query + }, + } + } + execute(query: Query) { + this.events.push({ + table: query.table, + action: query.action, + values: structuredClone(query.values), + }) + if (this.failBefore?.(query)) throw new Error('write failed') + const table = this.rows[query.table] + let result = table + .filter((row) => query.filters.every((filter) => filter(row))) + .sort((a, b) => a.id - b.id) + const count = result.length + if (query.action === 'insert') { + const row = { + id: Math.max(0, ...table.map((value) => value.id)) + 1, + ...structuredClone(query.values), + } + table.push(row) + result = [row] + } else if (query.action === 'update') { + for (const row of result) Object.assign(row, structuredClone(query.values)) + } + if (this.failAfter?.(query)) throw new Error('create outcome unknown') + if (query.action === 'read') + result = result.slice(query.start, Math.min(query.end + 1, query.start + this.pageLimit)) + if (query.one && result.length !== 1) throw new Error('Expected one result') + return { + data: structuredClone(query.one ? result[0] : result), + count: query.counted ? count : null, + } + } +} +const account = { mid: '1001', name: '当前名称' } +const config = SourceConfigSchema.parse({ + provider: 'd089e2c4-e3ec-42a6-953b-42530a726ac2', + account_mid: account.mid, + folder_ids: ['900123'], +}) +function page(): FavoritePage { + return { + folder: { id: '900123', title: '技术收藏', reported_count: 1 }, + owner: account, + items: [ + { + video: { aid: '11' }, + uploader: { mid: '2002', name: 'UP 主' }, + membership: { availability: 'unavailable', observed_at: '2026-11-07T00:00:00.000Z' }, + }, + ], + hasMore: false, + unsupported: 0, + unresolved: 0, + totalItems: 1, + ids: ['2:11'], + } +} +function source() { + return { + id: 1, + type: sourceType, + nickname: 'B 站', + config, + state: { kept: true }, + block: null, + } as unknown as Source +} +function next(source: Source) { + return advanceProgress(startProgress(source.state, config), '900123', page(), 1, 1) +} +function block(id: number, resolver: string, content: Values): Row { + return { id, resolver, storage: null, content: JSON.stringify(content, null, 2) } +} +function attach(context: { after: (action: () => void) => void }, database: Database) { + const originals = [Block.dbApi, Relation.dbApi, Source.dbApi] + Block.dbApi = database.client('blocks') as unknown as typeof Block.dbApi + Relation.dbApi = database.client('relations') as unknown as typeof Relation.dbApi + Source.dbApi = database.client('sources') as unknown as typeof Source.dbApi + context.after(() => { + ;[Block.dbApi, Relation.dbApi, Source.dbApi] = originals as typeof originals + }) +} +function savedGraph(database: Database, value: Source) { + const folder = database.rows.blocks.find((row) => row.resolver === resolverTypes.folder)! + const video = database.rows.blocks.find( + (row) => row.resolver === resolverTypes.video && JSON.parse(String(row.content)).aid === '11' + )! + const memberLink = database.rows.relations.find( + (row) => row.from_ === folder.id && row.content === 'contains' + )! + const item = database.rows.relations.find( + (row) => row.from_ === memberLink.to_ && row.content === 'item' + )! + assert.equal(item.to_, video.id) + const anchor = database.rows.blocks.find((row) => row.id === value.block)! + assert.equal(anchor.resolver, 'core.source.v1') + assert.equal(JSON.parse(String(anchor.content)).id, value.id) +} + +void test('ordinary pretty-printed JSON from another Peer is reused and repeat collection preserves IDs and user relations', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + value.block = 15 + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.rows.blocks.push( + block(10, resolverTypes.user, { + mid: '1001', + name: '原名称', + avatar_url: 'https://example.com/avatar.png', + }), + block(11, resolverTypes.folder, { id: '900123', title: '旧夹名' }), + block(12, resolverTypes.video, { aid: '11', title: '已知标题', description: '已知简介' }), + block(13, resolverTypes.user, { mid: '2002', name: 'UP 主' }), + block(14, resolverTypes.membership, { + availability: 'available', + favored_at: '2025-01-01T00:00:00.000Z', + observed_at: '2025-01-01T00:00:00.000Z', + }), + block(15, 'core.source.v1', { id: 1, type: sourceType, nickname: 'B 站' }), + { id: 16, resolver: 'core.text.v1', storage: null, content: '用户笔记' }, + block(17, resolverTypes.user, { mid: '11001', name: '提到 1001 的另一个人' }), + block(18, resolverTypes.video, { aid: '111', title: '提到 11 的另一个视频' }) + ) + database.rows.relations.push( + { id: 1, from_: 10, to_: 11, content: 'owns' }, + { id: 2, from_: 11, to_: 14, content: 'contains' }, + { id: 3, from_: 14, to_: 12, content: 'item' }, + { id: 4, from_: 12, to_: 13, content: 'author' }, + { id: 5, from_: 15, to_: 10, content: 'collects' }, + { id: 6, from_: 12, to_: 16, content: 'user:note' } + ) + const ids = database.rows.blocks.map((row) => row.id) + const userEdge = structuredClone(database.rows.relations[5]) + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + assert.deepEqual( + database.rows.blocks.map((row) => row.id), + ids + ) + assert.deepEqual( + database.rows.relations.find((row) => row.id === 6), + userEdge + ) + assert.equal(database.rows.relations.length, 6) + assert.equal( + JSON.parse(String(database.rows.blocks.find((row) => row.id === 10)!.content)).avatar_url, + 'https://example.com/avatar.png' + ) + assert.equal( + JSON.parse(String(database.rows.blocks.find((row) => row.id === 12)!.content)).title, + '已知标题' + ) + assert.equal( + JSON.parse(String(database.rows.blocks.find((row) => row.id === 14)!.content)).favored_at, + '2025-01-01T00:00:00.000Z' + ) + assert.equal(value.block, 15) + savedGraph(database, value) +}) + +void test('an existing membership for another video is never reused', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.rows.blocks.push( + block(10, resolverTypes.folder, { id: '900123' }), + block(11, resolverTypes.video, { aid: '99' }), + block(12, resolverTypes.membership, { + availability: 'available', + observed_at: '2025-01-01T00:00:00.000Z', + }) + ) + database.rows.relations.push( + { id: 1, from_: 10, to_: 12, content: 'contains' }, + { id: 2, from_: 12, to_: 11, content: 'item' } + ) + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + const currentVideo = database.rows.blocks.find( + (row) => row.resolver === resolverTypes.video && JSON.parse(String(row.content)).aid === '11' + )! + const currentItem = database.rows.relations.find( + (row) => row.content === 'item' && row.to_ === currentVideo.id + )! + assert.notEqual(currentItem.from_, 12) + assert.equal(database.rows.relations.find((row) => row.id === 2)!.to_, 11) +}) + +void test('partial graph failure leaves progress unchanged and the next execution repairs the ordinary graph', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.failBefore = (query) => + query.table === 'relations' && query.action === 'insert' && query.values.content === 'contains' + await assert.rejects( + new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)), + /write failed/ + ) + assert.deepEqual(value.state, { kept: true }) + assert.deepEqual(database.rows.sources[0]!.state, { kept: true }) + const originalVideo = database.rows.blocks.find((row) => row.resolver === resolverTypes.video)!.id + const incompleteMember = database.rows.blocks.find( + (row) => row.resolver === resolverTypes.membership + )!.id + assert.ok( + database.rows.relations.some((row) => row.from_ === incompleteMember && row.content === 'item') + ) + database.failBefore = null + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + assert.equal( + database.rows.blocks.find((row) => row.resolver === resolverTypes.video)!.id, + originalVideo + ) + assert.ok(database.rows.blocks.some((row) => row.id === incompleteMember)) + assert.ok(Object.hasOwn(value.state, 'bilibili')) + savedGraph(database, value) +}) + +void test('a lost create response is not replayed and a later execution reuses the visible native object', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.failAfter = (query) => + query.table === 'blocks' && + query.action === 'insert' && + query.values.resolver === resolverTypes.video + await assert.rejects( + new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)), + /create outcome unknown/ + ) + const created = database.rows.blocks.find((row) => row.resolver === resolverTypes.video)!.id + assert.deepEqual(value.state, { kept: true }) + const creates = () => + database.events.filter( + (event) => event.action === 'insert' && event.values.resolver === resolverTypes.video + ).length + assert.equal(creates(), 1) + database.failAfter = null + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + assert.equal(creates(), 1) + assert.equal( + database.rows.blocks.find((row) => row.resolver === resolverTypes.video)!.id, + created + ) + savedGraph(database, value) +}) + +void test('failure of the final Source update reuses the already saved graph and unbound Source anchor on the next execution', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.failBefore = (query) => query.table === 'sources' && query.action === 'update' + await assert.rejects( + new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)), + /write failed/ + ) + assert.deepEqual(value.state, { kept: true }) + assert.equal(value.block, null) + const ids = database.rows.blocks.map((row) => row.id) + const relations = structuredClone(database.rows.relations) + database.failBefore = null + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + assert.deepEqual( + database.rows.blocks.map((row) => row.id), + ids + ) + assert.deepEqual(database.rows.relations, relations) + savedGraph(database, value) +}) + +void test('normal paginated graph reads find existing objects beyond the service row cap', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + for (let id = 1; id <= 1100; id++) + database.rows.blocks.push(block(id, resolverTypes.user, { mid: String(1000000 + id) })) + database.rows.blocks.push(block(1101, resolverTypes.user, account)) + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + const users = database.rows.blocks.filter( + (row) => + row.resolver === resolverTypes.user && JSON.parse(String(row.content)).mid === account.mid + ) + assert.deepEqual( + users.map((row) => row.id), + [1101] + ) +}) diff --git a/packages/core/src/database/database.generated.ts b/packages/core/src/database/database.generated.ts index 45c386dd..1273484e 100644 --- a/packages/core/src/database/database.generated.ts +++ b/packages/core/src/database/database.generated.ts @@ -253,32 +253,6 @@ export type Database = { }, ] } - collection_records: { - Row: { - block: number - namespace: string - native_key: string - } - Insert: { - block: number - namespace: string - native_key: string - } - Update: { - block?: number - namespace?: string - native_key?: string - } - Relationships: [ - { - foreignKeyName: 'collection_records_block_fkey' - columns: ['block'] - isOneToOne: false - referencedRelation: 'blocks' - referencedColumns: ['id'] - }, - ] - } configs: { Row: { created_at: string @@ -881,19 +855,6 @@ export type Database = { [_ in never]: never } Functions: { - commit_source_batch: { - Args: { - expected_config: Json - expected_state: Json - namespace: string - next_state: Json - records: Json - relations: Json - root_key?: string - source_id: number - } - Returns: Json - } read_storage_blob: { Args: { blob_id: string }; Returns: unknown } renew_peer_lease: { Args: { peer: string; ttl_seconds: number } diff --git a/packages/core/src/database/runtime-contract.generated.json b/packages/core/src/database/runtime-contract.generated.json index ac2f0bd0..d9a1bc19 100644 --- a/packages/core/src/database/runtime-contract.generated.json +++ b/packages/core/src/database/runtime-contract.generated.json @@ -1,6 +1,6 @@ { "format": 1, - "revision": "source-batch-persistence-v1", + "revision": "peer-jwt-signing-defaults-v1", "protocol": { "format": 1, "schema": "inkcre" diff --git a/scripts/check-local-runtime-contract.mjs b/scripts/check-local-runtime-contract.mjs index ac7dbb30..022cefff 100644 --- a/scripts/check-local-runtime-contract.mjs +++ b/scripts/check-local-runtime-contract.mjs @@ -138,7 +138,7 @@ for (const legacyExport of ['httpAdapter', 'envAdapter', 'createEnvAdapter']) { } } -const localTargets = svcConfig.dev?.resources ?? svcConfig.dev?.targets +const localTargets = svcConfig.dev?.targets if (!localTargets) { errors.push('svc.json must declare development targets') } else { @@ -173,17 +173,18 @@ if (!localTargets) { if (target.scope !== 'worktree') { errors.push(`SVC target "${name}" must be worktree-scoped`) } - const probe = target.observations?.ready?.probe ?? target.probe - const provision = target.actions?.ensure ?? target.provision - if (probe?.kind !== 'exec' || JSON.stringify(probe.argv) !== JSON.stringify(expected.probe)) { + if ( + target.probe?.kind !== 'exec' || + JSON.stringify(target.probe.argv) !== JSON.stringify(expected.probe) + ) { errors.push(`SVC target "${name}" must use the identity probe that proves dev.instance`) } - if (JSON.stringify(provision?.argv) !== JSON.stringify(expected.command)) { + if (JSON.stringify(target.provision?.argv) !== JSON.stringify(expected.command)) { errors.push(`SVC target "${name}" must provision through scripts/dev.mjs`) } if ( expected.environment && - JSON.stringify(provision?.env) !== JSON.stringify(expected.environment) + JSON.stringify(target.provision?.env) !== JSON.stringify(expected.environment) ) { errors.push(`SVC target "${name}" must commit only the portable local provider default`) } @@ -236,10 +237,7 @@ for (const required of [ if (remoteCompose.includes('eval ') || !remoteCompose.includes('"$docker_bin" compose')) { errors.push('remote Compose runner must preserve argv without eval or shell command construction') } -const committedDatabaseEnvironment = - svcConfig.dev.resources?.database?.actions?.ensure?.env ?? - svcConfig.dev.targets?.database?.provision?.env ?? - {} +const committedDatabaseEnvironment = svcConfig.dev.targets.database.provision.env if ( committedDatabaseEnvironment?.INKCRE_DATABASE_SSH_TARGET || committedDatabaseEnvironment?.INKCRE_DATABASE_SSH_DOCKER_BIN || diff --git a/scripts/database-contract-source.mjs b/scripts/database-contract-source.mjs deleted file mode 100644 index 2d88a403..00000000 --- a/scripts/database-contract-source.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import { projectDatabaseCompatibilityContract } from './core-release-lib.mjs' - -/** Local development may consume an owner runtime without inventing a published release. */ -export function validateDevelopmentContract(descriptor, contract) { - if (descriptor.profile !== 'development' || descriptor.owner_repository !== 'InKCre/impl-py') { - throw new Error('local type generation requires an impl-py development runtime') - } - if (descriptor.converging) throw new Error('database owner is still converging') - if (contract.revision !== descriptor.contract_revision) { - throw new Error('local runtime and exported source contract revisions differ') - } - if (!contract.migration_heads?.includes(descriptor.migration_head)) { - throw new Error('local runtime and exported source migration heads differ') - } - return projectDatabaseCompatibilityContract(contract) -} diff --git a/scripts/database-provider.mjs b/scripts/database-provider.mjs index 732e848d..1ce0b079 100644 --- a/scripts/database-provider.mjs +++ b/scripts/database-provider.mjs @@ -34,11 +34,7 @@ function optionalJson(path) { } function provisionEnvironment(config) { - return ( - config.dev?.resources?.database?.actions?.ensure?.env ?? - config.dev?.targets?.database?.provision?.env ?? - {} - ) + return config.dev?.targets?.database?.provision?.env ?? {} } function declaredProviderEnvironment() { diff --git a/scripts/doctor.mjs b/scripts/doctor.mjs index 6eb8158f..984b9d8f 100644 --- a/scripts/doctor.mjs +++ b/scripts/doctor.mjs @@ -70,7 +70,7 @@ record( 'docs/_shared must be initialized' ) -const localTargets = svcConfig.dev?.resources ?? svcConfig.dev?.targets +const localTargets = svcConfig.dev?.targets record( 'svc-dev-targets', localTargets?.web?.scope === 'worktree' && diff --git a/scripts/prepare-native-checkout.mjs b/scripts/prepare-native-checkout.mjs deleted file mode 100644 index c25d98a3..00000000 --- a/scripts/prepare-native-checkout.mjs +++ /dev/null @@ -1,185 +0,0 @@ -import { execFile } from 'node:child_process' -import { createHash } from 'node:crypto' -import { lstat, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises' -import { dirname, join, resolve, sep } from 'node:path' -import { pathToFileURL } from 'node:url' -import { promisify } from 'node:util' -import semver from 'semver' - -const digest = (value) => createHash('sha256').update(value).digest('hex') -const executeFile = promisify(execFile) -const execute = async (command, args, cwd) => - ( - await executeFile(command, args, { - cwd, - encoding: 'utf8', - }) - ).stdout - -const listedFiles = ['ls-files', '-z', '--cached', '--others', '--exclude-standard'] -function includeSource(file) { - if (!file || file.startsWith('.runtime/')) return false - const dotenv = /(^|\/)\.env(?:\.|$)/.test(file) - return !dotenv || /(^|\/)\.env(?:\.[^/]*)?\.(example|sample)$/.test(file) -} - -async function packages(root) { - const result = new Map() - for (const family of ['packages', 'apps', 'extensions']) { - for (const entry of await readdir(join(root, family), { withFileTypes: true }).catch( - () => [] - )) { - if (!entry.isDirectory()) continue - const path = join(family, entry.name, 'package.json') - const text = await readFile(join(root, path), 'utf8').catch(() => null) - if (text) { - const metadata = JSON.parse(text) - result.set(metadata.name, { path, metadata }) - } - } - } - return result -} - -async function intents(root) { - return (await readdir(join(root, '.changeset')).catch(() => [])) - .filter((name) => name.endsWith('.md') && name.toLowerCase() !== 'readme.md') - .sort() -} - -async function verifySource(source, files, run) { - const listed = String(await run('git', listedFiles, source)) - .split('\0') - .filter(includeSource) - const currentFiles = [] - for (const file of listed) { - const stat = await lstat(join(source, file)).catch(() => null) - if (stat?.isFile()) currentFiles.push(file) - } - if (JSON.stringify(currentFiles.sort()) !== JSON.stringify(Object.keys(files).sort())) - throw new Error('Source file set changed after snapshot; refresh the candidate') - for (const [file, hash] of Object.entries(files)) { - const current = await readFile(join(source, file)).catch(() => null) - if (!current || digest(current) !== hash) - throw new Error( - `Source changed after snapshot: ${file}; refresh the candidate without reverting source work` - ) - } -} - -/** Prepare validation coordinates with the existing Changesets command; never edit source versions. */ -export async function prepareNativeCheckout({ - sourceDirectory, - outputDirectory, - offline = false, - run = execute, -}) { - const source = await realpath(sourceDirectory) - const shallow = String(await run('git', ['rev-parse', '--is-shallow-repository'], source)).trim() - if (shallow === 'true') - throw new Error( - 'Version preparation requires complete source history; fetch the selected checkout with fetch-depth: 0' - ) - const output = resolve(outputDirectory) - if (output === source || source.startsWith(output + sep)) - throw new Error('Candidate cannot replace the source or its ancestor') - if ((await readdir(output).catch(() => [])).length) - throw new Error('Candidate output must be empty') - const candidate = join(output, 'workspace') - await mkdir(candidate, { recursive: true }) - const revision = String(await run('git', ['rev-parse', 'HEAD'], source)).trim() - if (!/^[a-f0-9]{40}$/.test(revision)) throw new Error('Source requires one existing Git revision') - const sourceModified = !!String(await run('git', ['status', '--porcelain'], source)).trim() - const paths = String(await run('git', listedFiles, source)) - .split('\0') - .filter(Boolean) - .sort() - const files = {} - for (const file of paths) { - if (!includeSource(file)) continue - const origin = join(source, file) - const stat = await lstat(origin).catch(() => null) - if (!stat || stat.isDirectory()) continue // Git submodule content is not part of git archive either. - if (stat.isSymbolicLink()) throw new Error(`Candidate source contains a symbolic link: ${file}`) - const bytes = await readFile(origin) - const target = join(candidate, file) - await mkdir(dirname(target), { recursive: true }) - await writeFile(target, bytes, { mode: stat.mode & 0o777 }) - files[file] = digest(bytes) - } - const before = await packages(source) - const pending = await intents(source) - const sourceSnapshot = digest(JSON.stringify(files)) - /** @type {Array<[string, string[]]>} */ - const steps = [ - ['git', ['init', '--quiet']], - ['git', ['fetch', '--quiet', '--no-tags', source, `${revision}:refs/remotes/origin/main`]], - ['git', ['update-ref', 'refs/heads/main', revision]], - ['git', ['symbolic-ref', 'HEAD', 'refs/heads/main']], - ['git', ['read-tree', revision]], - ['pnpm', ['install', '--frozen-lockfile', ...(offline ? ['--offline'] : [])]], - ['pnpm', ['release:version']], - ['pnpm', ['install', '--lockfile-only', '--ignore-scripts', ...(offline ? ['--offline'] : [])]], - ] - await mkdir(join(output, 'logs')) - for (let index = 0; index < steps.length; index++) { - const [command, args] = steps[index] - const text = await run(command, args, candidate) - await writeFile(join(output, 'logs', `${index + 1}-${command}.log`), String(text ?? '')) - } - const after = await packages(candidate) - const coreVersion = after.get('@inkcre/core')?.metadata.version - if (!semver.valid(coreVersion)) throw new Error('Candidate Core SDK version is invalid') - for (const [name, previous] of before) { - const next = after.get(name) - if (!next) throw new Error(`Candidate lost workspace package ${name}`) - if (JSON.stringify(previous.metadata.inkcre) !== JSON.stringify(next.metadata.inkcre)) - throw new Error(`Candidate changed native identity or Host SDK range for ${name}`) - const association = next.metadata.inkcre?.module_federation - if (association && !semver.satisfies(coreVersion, association.host_sdk_version)) - throw new Error( - `Candidate versions are not prepared: ${name} requires ${association.host_sdk_version}, Core is ${coreVersion}` - ) - } - const remaining = await intents(candidate) - if (remaining.length) - throw new Error(`Candidate still has unconsumed version intent: ${remaining.join(', ')}`) - await verifySource(source, files, run) - const summary = { - kind: 'isolated-native-validation', - source, - candidate, - source_revision: revision, - source_modified: sourceModified, - source_snapshot_sha256: sourceSnapshot, - prepared_at: new Date().toISOString(), - source_changesets: pending, - versions: Object.fromEntries( - [...after].map(([name, value]) => [ - name, - { - source: before.get(name)?.metadata.version, - prepared: value.metadata.version, - }, - ]) - ), - } - await writeFile( - join(output, 'source-snapshot.json'), - JSON.stringify({ ...summary, files }, null, 2) + '\n' - ) - await writeFile(join(output, 'candidate.json'), JSON.stringify(summary, null, 2) + '\n') - return summary -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - const argument = (name) => process.argv[process.argv.indexOf(name) + 1] - if (!process.argv.includes('--output')) - throw new Error('Use --output ') - const summary = await prepareNativeCheckout({ - sourceDirectory: process.argv.includes('--source') ? argument('--source') : process.cwd(), - outputDirectory: argument('--output'), - offline: process.argv.includes('--offline'), - }) - console.log(JSON.stringify(summary)) -} diff --git a/scripts/sync-database-contract.mjs b/scripts/sync-database-contract.mjs index 801fa7e3..7cd29b4e 100644 --- a/scripts/sync-database-contract.mjs +++ b/scripts/sync-database-contract.mjs @@ -2,7 +2,6 @@ import { execFileSync } from 'node:child_process' import { createHash } from 'node:crypto' import { readFile, writeFile } from 'node:fs/promises' import { fileURLToPath } from 'node:url' -import { dirname, resolve } from 'node:path' import { inspectCoreRelease, @@ -18,37 +17,19 @@ import { stopDatabaseRuntime, waitForRuntime, } from './database-runtime-lib.mjs' -import { - readExternalDatabaseDescriptor, - resolveDatabaseProviderConfig, - runDatabaseDocker, -} from './database-provider.mjs' -import { validateDevelopmentContract } from './database-contract-source.mjs' +import { resolveDatabaseProviderConfig, runDatabaseDocker } from './database-provider.mjs' const repoRoot = fileURLToPath(new URL('..', import.meta.url)) const args = process.argv.slice(2) const imageIndex = args.indexOf('--image') const requestedImage = imageIndex >= 0 ? args[imageIndex + 1] : null -const runtimeIndex = args.indexOf('--runtime') -const contractIndex = args.indexOf('--runtime-contract') -const requestedRuntime = runtimeIndex >= 0 ? args[runtimeIndex + 1] : null -const requestedContract = contractIndex >= 0 ? args[contractIndex + 1] : null const check = args.includes('--check') -if ((runtimeIndex >= 0 || contractIndex >= 0) && (!requestedRuntime || !requestedContract)) { - throw new Error('local type generation requires --runtime and --runtime-contract together') -} -if (requestedRuntime && requestedImage) { - throw new Error('--runtime cannot be combined with --image') -} const config = validateCoreReleaseConfig(await readJson(`${repoRoot}/contracts/core-release.json`)) -const provider = requestedRuntime ? null : resolveDatabaseProviderConfig() +const provider = resolveDatabaseProviderConfig() const runDocker = (dockerArgs, options) => runDatabaseDocker(provider, dockerArgs, options) -const release = requestedRuntime - ? null - : requestedImage - ? inspectCoreRelease(runDocker, requestedImage, { allowLocalImage: true }) - : resolveConfiguredCoreRelease(runDocker, config) -const sourceLabel = release?.image ?? 'explicit local development runtime' +const release = requestedImage + ? inspectCoreRelease(runDocker, requestedImage, { allowLocalImage: true }) + : resolveConfiguredCoreRelease(runDocker, config) const instance = `typegen-${createHash('sha256') .update(`${repoRoot}/${process.pid}/${Date.now()}`) .digest('hex') @@ -97,38 +78,20 @@ function generateTypes(state, credentials) { async function writeOrCompare(path, content, label) { if (check) { const checked = await readFile(path) - if (!checked.equals(content)) throw new Error(`${label} is stale for ${sourceLabel}`) + if (!checked.equals(content)) throw new Error(`${label} is stale for ${release.image}`) return } await writeFile(path, content) } try { - let state - let credentials - let compatibility - if (requestedRuntime) { - const descriptorPath = resolve(requestedRuntime) - state = readExternalDatabaseDescriptor({ descriptor: descriptorPath }) - const contract = await readJson(resolve(requestedContract)) - compatibility = validateDevelopmentContract(state, contract) - credentials = await readJson(`${dirname(descriptorPath)}/credential.json`) - } else { - process.env.INKCRE_CORE_IMAGE = release.image - state = await ensureDatabaseRuntime(instance, { stdio: 'ignore' }) - runtimeStarted = true - await waitForRuntime(state) - credentials = await runtimeCredentials(instance) - compatibility = projectDatabaseCompatibilityContract(release.runtime_contract) - } + process.env.INKCRE_CORE_IMAGE = release.image + const state = await ensureDatabaseRuntime(instance, { stdio: 'ignore' }) + runtimeStarted = true + await waitForRuntime(state) + const credentials = await runtimeCredentials(instance) - let rawTypes - try { - rawTypes = generateTypes(state, credentials) - } catch { - // Child-process errors include argv, which contains the database password. - throw new Error('Supabase schema generation failed; inspect the development database owner') - } + const rawTypes = generateTypes(state, credentials) const generatedTypes = Buffer.from( output(workspaceBinary('oxfmt'), ['--stdin-filepath', 'database.generated.ts'], { input: rawTypes, @@ -141,7 +104,7 @@ try { ) const generatedRuntimeContract = Buffer.from( output(workspaceBinary('oxfmt'), ['--stdin-filepath', 'runtime-contract.generated.json'], { - input: stableJson(compatibility), + input: stableJson(projectDatabaseCompatibilityContract(release.runtime_contract)), }) ) await writeOrCompare( @@ -150,7 +113,7 @@ try { 'generated runtime contract' ) console.log( - `[contract] ${check ? 'verified' : 'synchronized'} ${sourceLabel} through Supabase CLI` + `[contract] ${check ? 'verified' : 'synchronized'} ${release.image} through Supabase CLI` ) } finally { if (runtimeStarted) { diff --git a/scripts/tests/database-contract-source.test.mjs b/scripts/tests/database-contract-source.test.mjs deleted file mode 100644 index bfc54cac..00000000 --- a/scripts/tests/database-contract-source.test.mjs +++ /dev/null @@ -1,38 +0,0 @@ -import assert from 'node:assert/strict' -import { test } from 'node:test' -import { validateDevelopmentContract } from '../database-contract-source.mjs' - -const descriptor = { - profile: 'development', - owner_repository: 'InKCre/impl-py', - contract_revision: 'source-batch-persistence-v1', - migration_head: 'batch-head', -} -const contract = { - format: 1, - revision: descriptor.contract_revision, - migration_heads: ['batch-head'], - protocol: { format: 1, schema: 'inkcre' }, - jwt: { algorithm: 'HS256', role: 'authenticated', issuer: 'inkcre-peer', audience: 'inkcre-api' }, -} - -void test('local type generation projects the matching owner contract without release provenance', () => { - assert.deepEqual(validateDevelopmentContract(descriptor, contract), { - format: 1, - revision: descriptor.contract_revision, - protocol: contract.protocol, - jwt: contract.jwt, - }) -}) - -void test('local type generation rejects mismatched owner state and source contracts', () => { - for (const change of [ - { profile: 'production' }, - { owner_repository: 'other' }, - { converging: true }, - { contract_revision: 'old-revision' }, - { migration_head: 'old-head' }, - ]) { - assert.throws(() => validateDevelopmentContract({ ...descriptor, ...change }, contract)) - } -}) diff --git a/scripts/tests/prepare-native-checkout.test.mjs b/scripts/tests/prepare-native-checkout.test.mjs deleted file mode 100644 index 0239f151..00000000 --- a/scripts/tests/prepare-native-checkout.test.mjs +++ /dev/null @@ -1,278 +0,0 @@ -import assert from 'node:assert/strict' -import { execFileSync } from 'node:child_process' -import { mkdtemp, mkdir, readFile, rm, writeFile, unlink } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { fileURLToPath } from 'node:url' -import test from 'node:test' -import { prepareNativeCheckout } from '../prepare-native-checkout.mjs' - -const repository = fileURLToPath(new URL('../..', import.meta.url)) - -async function fixture() { - const root = await mkdtemp(join(tmpdir(), 'inkcre-native-checkout-')) - const source = join(root, 'source') - const output = join(root, 'candidate') - const input = { - 'package.json': { name: 'fixture', private: true }, - 'packages/core/package.json': { name: '@inkcre/core', version: '0.4.4' }, - 'extensions/example/package.json': { - name: '@inkcre/example', - version: '0.1.0', - inkcre: { - name: 'inkcre/example', - module_federation: { host_sdk: '@inkcre/core', host_sdk_version: '>=0.4.5 <0.5.0' }, - }, - }, - '.changeset/compatible.md': '---\n"@inkcre/core": patch\n---\nCompatible capability\n', - } - for (const [file, value] of Object.entries(input)) { - await mkdir(join(source, file, '..'), { recursive: true }) - await writeFile(join(source, file), typeof value === 'string' ? value : JSON.stringify(value)) - } - const calls = [] - const run = async (command, args, cwd) => { - calls.push({ command, args, cwd }) - if (command === 'git' && args.includes('--is-shallow-repository')) return 'false' - if (command === 'git' && args[0] === 'rev-parse') return 'a'.repeat(40) - if (command === 'git' && args[0] === 'status') return ' M packages/core/package.json\n' - if (command === 'git' && args[0] === 'ls-files') return Object.keys(input).join('\0') + '\0' - if (command === 'pnpm' && args[0] === 'release:version') { - await writeFile( - join(cwd, 'packages/core/package.json'), - JSON.stringify({ name: '@inkcre/core', version: '0.4.5' }) - ) - await unlink(join(cwd, '.changeset/compatible.md')) - } - return '' - } - return { root, source, output, calls, run, input } -} - -void test('the existing version command prepares an isolated candidate without consuming source intent', async () => { - const context = await fixture() - try { - const result = await prepareNativeCheckout({ - sourceDirectory: context.source, - outputDirectory: context.output, - run: context.run, - offline: true, - }) - assert.equal(result.versions['@inkcre/core'].prepared, '0.4.5') - assert.equal( - JSON.parse(await readFile(join(context.source, 'packages/core/package.json'), 'utf8')) - .version, - '0.4.4' - ) - assert.match( - await readFile(join(context.source, '.changeset/compatible.md'), 'utf8'), - /Compatible capability/ - ) - assert.ok( - context.calls.some( - (call) => - call.command === 'pnpm' && - call.args[0] === 'release:version' && - call.cwd === result.candidate - ) - ) - assert.ok( - context.calls.some((call) => call.command === 'pnpm' && call.args.includes('--offline')) - ) - assert.equal( - context.calls.some((call) => call.args[0] === 'commit'), - false - ) - assert.equal(result.source_modified, true) - } finally { - await rm(context.root, { recursive: true, force: true }) - } -}) - -void test('committed version intent is resolved by the actual CLI from complete isolated Git history', async () => { - const context = await fixture() - const run = async (command, args, cwd) => { - if (command === 'pnpm' && args[0] === 'install') return '' - return execFileSync( - command === 'pnpm' ? join(repository, 'node_modules/.bin/changeset') : command, - command === 'pnpm' ? ['version'] : args, - { cwd, encoding: 'utf8', timeout: 15_000 } - ) - } - try { - await writeFile( - join(context.source, 'pnpm-workspace.yaml'), - 'packages:\n - packages/*\n - extensions/*\n' - ) - await writeFile( - join(context.source, '.changeset/config.json'), - JSON.stringify({ - changelog: '@changesets/cli/changelog', - commit: false, - fixed: [], - linked: [], - access: 'restricted', - baseBranch: 'main', - updateInternalDependencies: 'patch', - ignore: [], - }) - ) - execFileSync('git', ['init', '--quiet', '-b', 'main'], { cwd: context.source }) - execFileSync('git', ['add', '.'], { cwd: context.source }) - execFileSync( - 'git', - [ - '-c', - 'user.name=Fixture', - '-c', - 'user.email=fixture@example.test', - 'commit', - '--quiet', - '-m', - 'Initial native packages', - ], - { cwd: context.source } - ) - await writeFile( - join(context.source, 'README.md'), - 'Version intent is committed before candidate preparation.\n' - ) - execFileSync('git', ['add', 'README.md'], { cwd: context.source }) - execFileSync( - 'git', - [ - '-c', - 'user.name=Fixture', - '-c', - 'user.email=fixture@example.test', - 'commit', - '--quiet', - '-m', - 'Prepare source checkpoint', - ], - { cwd: context.source } - ) - const result = await prepareNativeCheckout({ - sourceDirectory: context.source, - outputDirectory: context.output, - run, - }) - assert.equal(result.source_modified, false) - assert.equal(result.versions['@inkcre/core'].prepared, '0.4.5') - assert.equal( - execFileSync('git', ['rev-parse', '--is-shallow-repository'], { - cwd: result.candidate, - encoding: 'utf8', - }).trim(), - 'false' - ) - assert.match( - await readFile(join(context.source, '.changeset/compatible.md'), 'utf8'), - /Compatible capability/ - ) - } finally { - await rm(context.root, { recursive: true, force: true }) - } -}) - -void test('a version command that leaves the Host below the producer minimum is rejected', async () => { - const context = await fixture() - try { - const run = async (command, args, cwd) => { - if (command === 'pnpm' && args[0] === 'release:version') { - await unlink(join(cwd, '.changeset/compatible.md')) - return '' - } - return context.run(command, args, cwd) - } - await assert.rejects( - prepareNativeCheckout({ - sourceDirectory: context.source, - outputDirectory: context.output, - run, - }), - /versions are not prepared/ - ) - } finally { - await rm(context.root, { recursive: true, force: true }) - } -}) - -void test('loosening a future producer range cannot masquerade as version preparation', async () => { - const context = await fixture() - try { - const run = async (command, args, cwd) => { - const result = await context.run(command, args, cwd) - if (command === 'pnpm' && args[0] === 'release:version') { - const file = join(cwd, 'extensions/example/package.json') - const value = JSON.parse(await readFile(file, 'utf8')) - value.inkcre.module_federation.host_sdk_version = '>=0.4.4 <0.5.0' - await writeFile(file, JSON.stringify(value)) - } - return result - } - await assert.rejects( - prepareNativeCheckout({ - sourceDirectory: context.source, - outputDirectory: context.output, - run, - }), - /changed native identity or Host SDK range/ - ) - } finally { - await rm(context.root, { recursive: true, force: true }) - } -}) - -void test('source changes during preparation invalidate the candidate without reverting the writer', async () => { - const context = await fixture() - try { - const run = async (command, args, cwd) => { - const result = await context.run(command, args, cwd) - if (command === 'pnpm' && args[0] === 'release:version') - await writeFile(join(context.source, '.changeset/compatible.md'), 'concurrent source work') - return result - } - await assert.rejects( - prepareNativeCheckout({ - sourceDirectory: context.source, - outputDirectory: context.output, - run, - }), - /Source changed after snapshot/ - ) - assert.equal( - await readFile(join(context.source, '.changeset/compatible.md'), 'utf8'), - 'concurrent source work' - ) - } finally { - await rm(context.root, { recursive: true, force: true }) - } -}) - -void test('new source files added during preparation require a fresh snapshot too', async () => { - const context = await fixture() - try { - const run = async (command, args, cwd) => { - const result = await context.run(command, args, cwd) - if (command === 'pnpm' && args[0] === 'release:version') { - context.input['extensions/example/new-source.ts'] = 'export const added = true\n' - await writeFile( - join(context.source, 'extensions/example/new-source.ts'), - context.input['extensions/example/new-source.ts'] - ) - } - return result - } - await assert.rejects( - prepareNativeCheckout({ - sourceDirectory: context.source, - outputDirectory: context.output, - run, - }), - /Source file set changed after snapshot/ - ) - } finally { - await rm(context.root, { recursive: true, force: true }) - } -}) diff --git a/svc.json b/svc.json index b8f07cd1..b76509d4 100644 --- a/svc.json +++ b/svc.json @@ -1,144 +1,84 @@ { - "schema_version": 5, + "schema_version": 4, "dev": { - "resources": { + "targets": { "database": { - "scope": "worktree", - "observations": { - "ready": { - "label": "就绪状态", - "purpose": "runtime-readiness", - "probe": { - "argv": ["node", "scripts/probe-database.mjs", "${dev.instance}"], - "cwd": ".", - "kind": "exec", - "output_limit": 2048, - "timeout": 5 - } - } + "poll_interval": 1, + "probe": { + "argv": ["node", "scripts/probe-database.mjs", "${dev.instance}"], + "cwd": ".", + "kind": "exec", + "output_limit": 2048, + "timeout": 5 }, - "actions": { - "ensure": { - "argv": ["node", "scripts/database-runtime.mjs", "ensure", "${dev.instance}"], - "cwd": ".", - "env": { - "INKCRE_DATABASE_PROVIDER": "local" - }, - "kind": "exec", - "intent": "准备database开发资源", - "traits": ["runtime", "provision"], - "effect_scope": "当前工作树的database开发资源", - "data_effect": "modify", - "completion": "exit", - "interruption": "wait", - "conditions": ["ready"], - "completion_timeout": 180, - "poll_interval": 1, - "convergent": true - } - } + "provision": { + "argv": ["node", "scripts/database-runtime.mjs", "ensure", "${dev.instance}"], + "cwd": ".", + "env": { + "INKCRE_DATABASE_PROVIDER": "local" + }, + "kind": "exec", + "mode": "run" + }, + "readiness_timeout": 180, + "scope": "worktree" }, "web": { - "scope": "worktree", - "observations": { - "ready": { - "label": "就绪状态", - "purpose": "runtime-readiness", - "probe": { - "argv": ["node", "scripts/probe-dev.mjs", "web", "${dev.instance}"], - "cwd": ".", - "kind": "exec", - "output_limit": 1024, - "timeout": 5 - } - } + "access": ["https://client-web-${dev.instance}.localhost/"], + "poll_interval": 0.5, + "probe": { + "argv": ["node", "scripts/probe-dev.mjs", "web", "${dev.instance}"], + "cwd": ".", + "kind": "exec", + "output_limit": 1024, + "timeout": 5 }, - "actions": { - "ensure": { - "argv": ["node", "scripts/dev.mjs", "web"], - "cwd": ".", - "kind": "exec", - "intent": "准备web开发资源", - "traits": ["runtime", "provision"], - "effect_scope": "当前工作树的web开发资源", - "data_effect": "modify", - "completion": "ready", - "interruption": "terminate", - "conditions": ["ready"], - "completion_timeout": 90, - "poll_interval": 0.5, - "convergent": true - } + "provision": { + "argv": ["node", "scripts/dev.mjs", "web"], + "cwd": ".", + "kind": "exec", + "mode": "run" }, - "access": ["https://client-web-${dev.instance}.localhost/"] + "readiness_timeout": 90, + "scope": "worktree" }, "web-ui": { - "scope": "worktree", - "observations": { - "ready": { - "label": "就绪状态", - "purpose": "runtime-readiness", - "probe": { - "argv": ["node", "scripts/probe-dev.mjs", "web-ui", "${dev.instance}"], - "cwd": ".", - "kind": "exec", - "output_limit": 1024, - "timeout": 5 - } - } + "access": ["https://client-web-ui-${dev.instance}.localhost/"], + "poll_interval": 0.5, + "probe": { + "argv": ["node", "scripts/probe-dev.mjs", "web-ui", "${dev.instance}"], + "cwd": ".", + "kind": "exec", + "output_limit": 1024, + "timeout": 5 }, - "actions": { - "ensure": { - "argv": ["node", "scripts/dev.mjs", "web-ui"], - "cwd": ".", - "kind": "exec", - "intent": "准备web-ui开发资源", - "traits": ["runtime", "provision"], - "effect_scope": "当前工作树的web-ui开发资源", - "data_effect": "modify", - "completion": "ready", - "interruption": "terminate", - "conditions": ["ready"], - "completion_timeout": 90, - "poll_interval": 0.5, - "convergent": true - } + "provision": { + "argv": ["node", "scripts/dev.mjs", "web-ui"], + "cwd": ".", + "kind": "exec", + "mode": "run" }, - "access": ["https://client-web-ui-${dev.instance}.localhost/"] + "readiness_timeout": 90, + "scope": "worktree" }, "webext": { - "scope": "worktree", - "observations": { - "ready": { - "label": "就绪状态", - "purpose": "runtime-readiness", - "probe": { - "argv": ["node", "scripts/probe-dev.mjs", "webext", "${dev.instance}"], - "cwd": ".", - "kind": "exec", - "output_limit": 1024, - "timeout": 5 - } - } + "access": ["https://webext-${dev.instance}.localhost/popup.html"], + "poll_interval": 0.5, + "probe": { + "argv": ["node", "scripts/probe-dev.mjs", "webext", "${dev.instance}"], + "cwd": ".", + "kind": "exec", + "output_limit": 1024, + "timeout": 5 }, - "actions": { - "ensure": { - "argv": ["node", "scripts/dev.mjs", "webext"], - "cwd": ".", - "kind": "exec", - "intent": "准备webext开发资源", - "traits": ["runtime", "provision"], - "effect_scope": "当前工作树的webext开发资源", - "data_effect": "modify", - "completion": "ready", - "interruption": "terminate", - "conditions": ["ready"], - "completion_timeout": 120, - "poll_interval": 0.5, - "convergent": true - } + "provision": { + "argv": ["node", "scripts/dev.mjs", "webext"], + "cwd": ".", + "kind": "exec", + "mode": "run" }, - "access": ["https://webext-${dev.instance}.localhost/popup.html"] + "readiness_timeout": 120, + "scope": "worktree" } } } From 9f02a8829437459fd8724bd86d0e514a6a82327b Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Sat, 7 Nov 2026 02:31:00 +0800 Subject: [PATCH 07/12] =?UTF-8?q?fix(ci):=20=E4=BF=AE=E6=AD=A3=20Linux=20?= =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8=E5=A4=B9=E5=85=B7=E5=90=AF=E5=8A=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 仅隔离 Linux CI 浏览器使用无 sandbox 启动参数 - 提前退出时立即报告原因，避免空等验收超时 --- scripts/verify-browser-automation.mjs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/verify-browser-automation.mjs b/scripts/verify-browser-automation.mjs index 30613500..c6fbcb14 100644 --- a/scripts/verify-browser-automation.mjs +++ b/scripts/verify-browser-automation.mjs @@ -167,6 +167,9 @@ throw new Error('Web fixture timed out'); executable, [ '--headless=new', + // Hosted Linux CI disallows Chromium's user-namespace sandbox. This browser + // uses a disposable profile and serves only this fixture's synthetic pages. + ...(process.platform === 'linux' && process.env.CI ? ['--no-sandbox'] : []), '--no-first-run', '--no-default-browser-check', '--disable-background-networking', @@ -185,6 +188,14 @@ throw new Error('Web fixture timed out'); webReport = { ok: false, error: String(error) } done() }) + browserProcess.on('exit', (code, signal) => { + if (webReport && nativeReport) return + webReport = { + ok: false, + error: `Browser fixture exited before reporting (code ${code}, signal ${signal})\n${stderr}`, + } + done() + }) await Promise.race([ finished, new Promise((_, reject) => { From 856273af1b2c1d239d821a9da8c0b3319132d345 Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Mon, 9 Nov 2026 15:58:28 +0800 Subject: [PATCH 08/12] =?UTF-8?q?fix(web):=20=E8=AE=A9=E6=AD=A3=E5=BC=8F?= =?UTF-8?q?=20Pages=20=E5=8F=AA=E6=9E=84=E5=BB=BA=20Web=20=E4=BA=A7?= =?UTF-8?q?=E7=89=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 沿用 Web 产物合同检查，避免待发布 MF 阻断首次 main 部署 - 保留全仓 CI 与各发行流程的 MF 兼容性检查 --- .github/workflows/pages-deploy.yml | 6 ++++-- docs/40-deployment/web-delivery.md | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pages-deploy.yml b/.github/workflows/pages-deploy.yml index c91a301c..88ed07cf 100644 --- a/.github/workflows/pages-deploy.yml +++ b/.github/workflows/pages-deploy.yml @@ -83,9 +83,11 @@ jobs: working-directory: release run: pnpm install --frozen-lockfile - - name: Build the exact-main release + - name: Build and check the exact-main Web App working-directory: release - run: pnpm build + run: | + pnpm --filter @inkcre/web... run build + pnpm check:release:web - name: Deploy to Cloudflare Pages id: pages diff --git a/docs/40-deployment/web-delivery.md b/docs/40-deployment/web-delivery.md index 198356e2..6426a949 100644 --- a/docs/40-deployment/web-delivery.md +++ b/docs/40-deployment/web-delivery.md @@ -46,9 +46,10 @@ into chunks and fetched on first navigation; repeat visits can reuse unchanged b [`.github/workflows/pages-deploy.yml`](../../.github/workflows/pages-deploy.yml) owns production delivery. A protected-`main` push selects that exact source SHA after strict required PR checks, while delivery rejects the run if a newer revision supersedes it. The delivery job checks out the -exact source, installs the frozen workspace, builds the release, reverifies `main`, and deploys -those same-run bytes to the Cloudflare Pages `main` branch in the protected `production` -environment. +exact source, installs the frozen workspace, and builds the Web App and its workspace dependencies. +It runs `pnpm check:release:web` before deploying those same-run bytes to the Cloudflare Pages +`main` branch in the protected `production` environment. Workspace CI checks Module Federation +compatibility, and first-party MF publication validates each selected Extension release. The workspace lock owns the exact Wrangler version, and Pages production/preview execute it through the pinned pnpm toolchain. Preview cleanup is intentionally different: its tombstone is a standalone From ad120e8511771d8a79d2b46913bcf324675e4d0b Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Mon, 9 Nov 2026 21:23:29 +0800 Subject: [PATCH 09/12] =?UTF-8?q?fix(ci):=20=E6=8C=89=E5=BE=85=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E7=89=88=E6=9C=AC=E8=AE=A1=E5=88=92=E6=9E=84=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 空 Changesets 计划直接构建现有版本 - 受信 Pages 预览工作流由前置 PR #197 交付 --- .github/workflows/ci.yml | 9 ++++++++- .github/workflows/pages-preview.yml | 8 -------- docs/40-deployment/native-extension-delivery.md | 6 +++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5eb306b..33ef8c07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,9 +113,16 @@ jobs: - name: 校验源代码格式、静态规则和类型 run: pnpm format:check && pnpm lint && pnpm type-check - - name: 在临时 checkout 中准备兼容构建版本 + - name: 读取待处理的版本计划 + id: build-versions run: | git show-ref --verify --quiet refs/heads/main || git branch main HEAD + pnpm exec changeset status --output "$RUNNER_TEMP/build-versions.json" + node -e 'console.log(`needed=${require(process.argv[1]).releases.length > 0}`)' "$RUNNER_TEMP/build-versions.json" >> "$GITHUB_OUTPUT" + + - name: 在临时 checkout 中准备兼容构建版本 + if: steps.build-versions.outputs.needed == 'true' + run: | pnpm release:version pnpm install --lockfile-only --ignore-scripts diff --git a/.github/workflows/pages-preview.yml b/.github/workflows/pages-preview.yml index d09ab775..dcd3e625 100644 --- a/.github/workflows/pages-preview.yml +++ b/.github/workflows/pages-preview.yml @@ -72,7 +72,6 @@ jobs: with: ref: ${{ needs.identity.outputs.head_sha }} path: candidate - fetch-depth: 0 persist-credentials: false - name: Install the pinned pnpm release @@ -103,13 +102,6 @@ jobs: env: PDM_CHECK_UPDATE: 'false' - - name: Prepare preview versions in the temporary checkout - working-directory: candidate - run: | - git show-ref --verify --quiet refs/heads/main || git branch main HEAD - pnpm release:version - pnpm install --lockfile-only --ignore-scripts - - name: Build the exact-head SPA and Module Federation snapshots working-directory: candidate run: pnpm build diff --git a/docs/40-deployment/native-extension-delivery.md b/docs/40-deployment/native-extension-delivery.md index bb158cc5..cb01a86e 100644 --- a/docs/40-deployment/native-extension-delivery.md +++ b/docs/40-deployment/native-extension-delivery.md @@ -71,9 +71,9 @@ copy so the preview can load a mutually compatible Host and MF when a change rai Host SDK. This produces temporary derived versions for preview only. It neither consumes the pending changesets in the PR nor replaces the normal Version PR and protected-main release lifecycle. -Workspace contract 先校验源代码格式、静态规则和类型，再直接在 CI 的临时 checkout 中运行 -`pnpm release:version` 与 `pnpm install --lockfile-only --ignore-scripts`，然后构建并验证 Host 和 MF。 -Pages preview 在受信控制器选定的 PR head 临时 checkout 中使用同样的正式版本命令，保留完整历史供 Changesets 读取。 +Workspace contract 先校验源代码格式、静态规则和类型，再读取 Changesets 的版本计划。有待升版的包时， +在 CI 的临时 checkout 中运行 `pnpm release:version` 与 `pnpm install --lockfile-only --ignore-scripts`，然后构建并验证 Host 和 MF。 +Pages preview 对受信控制器选定的 PR head 使用同样的版本计划和命令，保留完整历史供 Changesets 读取。 这些派生版本仅用于验证和预览，不回写源分支，也不替代 Version PR 或正式发布。 本地需要验证尚未发布的 SDK 时，在临时 clone 中运行上述命令及 `pnpm build`；日常工作树继续保留版本意图。 From 8a505744dd99b0e395018f8ee50b9b6dcdcceb45 Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Mon, 9 Nov 2026 21:24:52 +0800 Subject: [PATCH 10/12] =?UTF-8?q?fix(ci):=20=E6=98=8E=E7=A1=AE=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E8=AE=A1=E5=88=92=E7=9A=84=E6=AD=A5=E9=AA=A4=E8=BE=93?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 使用字符串拼接写出布尔值，使 ShellCheck 正确识别 Node 表达式 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33ef8c07..b4e14f8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: run: | git show-ref --verify --quiet refs/heads/main || git branch main HEAD pnpm exec changeset status --output "$RUNNER_TEMP/build-versions.json" - node -e 'console.log(`needed=${require(process.argv[1]).releases.length > 0}`)' "$RUNNER_TEMP/build-versions.json" >> "$GITHUB_OUTPUT" + node -e 'console.log("needed=" + (require(process.argv[1]).releases.length > 0))' "$RUNNER_TEMP/build-versions.json" >> "$GITHUB_OUTPUT" - name: 在临时 checkout 中准备兼容构建版本 if: steps.build-versions.outputs.needed == 'true' From 4c5438c80a9992fb656273bb3c6e54b3216b9eda Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Mon, 9 Nov 2026 21:28:23 +0800 Subject: [PATCH 11/12] =?UTF-8?q?fix(ci):=20=E9=87=8D=E8=AF=95=E6=B8=85?= =?UTF-8?q?=E7=90=86=20Chromium=20=E6=B5=8B=E8=AF=95=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 容忍 Chromium 子进程收尾期间测试 profile 的短暂目录变化 --- scripts/verify-browser-automation.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/verify-browser-automation.mjs b/scripts/verify-browser-automation.mjs index c6fbcb14..531d3955 100644 --- a/scripts/verify-browser-automation.mjs +++ b/scripts/verify-browser-automation.mjs @@ -220,5 +220,6 @@ throw new Error('Web fixture timed out'); if (browserProcess.exitCode === null) browserProcess.kill('SIGKILL') } for (const server of servers) server.close() - await rm(temporary, { recursive: true, force: true }) + // Chromium helpers can still flush the disposable profile after the main process exits. + await rm(temporary, { recursive: true, force: true, maxRetries: 5 }) } From 34396e499bbc8d31ac5e319af10cf181fe063d52 Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Mon, 9 Nov 2026 21:54:32 +0800 Subject: [PATCH 12/12] =?UTF-8?q?docs(shared):=20=E5=AF=B9=E9=BD=90=20B=20?= =?UTF-8?q?=E7=AB=99=E6=94=B6=E8=97=8F=E5=A4=B9=E5=85=B1=E4=BA=AB=E5=A5=91?= =?UTF-8?q?=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将共享文档引用更新到 Hub 主分支的合并提交 --- docs/_shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_shared b/docs/_shared index b50921b1..ad76242b 160000 --- a/docs/_shared +++ b/docs/_shared @@ -1 +1 @@ -Subproject commit b50921b19cb6cf81ad4c09786423195974d901b2 +Subproject commit ad76242bd3a10536001aa751c1152bec7fb42990