diff --git a/.changeset/bilibili-favorites.md b/.changeset/bilibili-favorites.md new file mode 100644 index 00000000..61103a4a --- /dev/null +++ b/.changeset/bilibili-favorites.md @@ -0,0 +1,5 @@ +--- +'@inkcre/ext-bilibili': minor +--- + +新增 B 站收藏夹插件，通过所选浏览器采集自己的公开和私密收藏夹，保存原生身份、文字资料和成员关系，并提供独立读取视图。支持有界分批与进度复核，不保存视频二进制。 diff --git a/.changeset/web-source-browser-automation.md b/.changeset/web-source-browser-automation.md new file mode 100644 index 00000000..388b6285 --- /dev/null +++ b/.changeset/web-source-browser-automation.md @@ -0,0 +1,9 @@ +--- +'@inkcre/core': patch +'@inkcre/web': patch +'@inkcre/webext': patch +--- + +为 Web Source 提供基于 Puppeteer 的本机浏览器能力，支持明确的 Web origin 授权、精确浏览器绑定、登录态页面交互与网络响应读取。 + +Chromium 扩展新增 debugger 权限，最低 Chrome 125。新建页与借用页分别清理，取消和断连不会自动重放操作。普通 Source Job 通过业务资格钩子选择执行 Peer。 diff --git a/.github/preview/extensions.json b/.github/preview/extensions.json index ccc70f2a..d7158886 100644 --- a/.github/preview/extensions.json +++ b/.github/preview/extensions.json @@ -1,6 +1,11 @@ { "schema_version": 1, "distributions": [ + { + "kind": "module_federation", + "producer": "../../extensions/bilibili/package.json", + "artifact": "../../extensions/bilibili/dist/web" + }, { "kind": "module_federation", "producer": "../../extensions/caldav/package.json", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6586dd07..b4e14f8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,8 +110,30 @@ jobs: - name: Install the frozen workspace run: pnpm install --frozen-lockfile - - name: 校验 TS7、Vue 类型、静态规则及产物合同 - run: pnpm check + - name: 校验源代码格式、静态规则和类型 + run: pnpm format:check && pnpm lint && pnpm type-check + + - name: 读取待处理的版本计划 + id: build-versions + run: | + git show-ref --verify --quiet refs/heads/main || git branch main HEAD + pnpm exec changeset status --output "$RUNNER_TEMP/build-versions.json" + node -e 'console.log("needed=" + (require(process.argv[1]).releases.length > 0))' "$RUNNER_TEMP/build-versions.json" >> "$GITHUB_OUTPUT" + + - name: 在临时 checkout 中准备兼容构建版本 + if: steps.build-versions.outputs.needed == 'true' + run: | + pnpm release:version + pnpm install --lockfile-only --ignore-scripts + + - name: 构建并校验相互兼容的 Host 与 MF 产物 + run: pnpm build + + - name: Verify independent SDK and Job observations + run: pnpm check:sdk && pnpm check:web + + - name: 验证 B 站收藏采集规则 + run: pnpm --filter @inkcre/ext-bilibili test - name: Verify independent MF publication run: >- @@ -120,6 +142,9 @@ jobs: scripts/tests/release-checkout.test.mjs scripts/tests/core-sdk-compatibility.test.mjs scripts/tests/pages-preview.test.mjs + scripts/tests/browser-automation-transport.test.mjs + scripts/tests/job-cancellation.test.mjs + scripts/tests/source-anchor.test.mjs peer-database: # Existing branch protection binds this exact check name. diff --git a/.github/workflows/pages-deploy.yml b/.github/workflows/pages-deploy.yml index c91a301c..88ed07cf 100644 --- a/.github/workflows/pages-deploy.yml +++ b/.github/workflows/pages-deploy.yml @@ -83,9 +83,11 @@ jobs: working-directory: release run: pnpm install --frozen-lockfile - - name: Build the exact-main release + - name: Build and check the exact-main Web App working-directory: release - run: pnpm build + run: | + pnpm --filter @inkcre/web... run build + pnpm check:release:web - name: Deploy to Cloudflare Pages id: pages diff --git a/.gitignore b/.gitignore index 39ff2a0b..755fefcb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ node_modules package-lock.json # Build output +.output/ +.wxt/ dist dist-ssr coverage diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index abef4e3c..d80f9469 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -832,3 +832,290 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice This Source Code Form is "Incompatible With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. + + +====================================================================== +puppeteer-core 25.16.0 +https://github.com/puppeteer/puppeteer + +Copyright 2017 Google Inc. +SPDX-License-Identifier: Apache-2.0 +The complete Apache-2.0 text is retained below with Puppeteer's bundled RxJS dependency. + + +====================================================================== +mitt (bundled by puppeteer-core 25.16.0) +MIT License + +Copyright (c) 2021 Jason Miller + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +====================================================================== +parsel-js (bundled by puppeteer-core 25.16.0) +MIT License + +Copyright (c) 2020 Lea Verou + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +====================================================================== +urlpattern-polyfill (bundled by puppeteer-core 25.16.0) +Copyright 2020 Intel Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +====================================================================== +rxjs (bundled by puppeteer-core 25.16.0) +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2015-2018 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/apps/web/src/core.ts b/apps/web/src/core.ts index 95fd7c8b..db3cd373 100644 --- a/apps/web/src/core.ts +++ b/apps/web/src/core.ts @@ -17,6 +17,7 @@ import { localStorageAdapter, registerCoreResolvers, TextResolver, + SourceResolver, MarkdownResolver, ResolverCache, AudioResolver, @@ -122,6 +123,7 @@ export function setupResolvers(): void { MarkdownResolver.previewRenderer = ContentPreview MarkdownResolver.solvedContentRenderer = ContentHtml TextResolver.previewRenderer = ContentPreview + SourceResolver.previewRenderer = ContentPreview AudioResolver.previewRenderer = ContentPreview EpubResolver.previewRenderer = ContentPreview FileResolver.previewRenderer = ContentPreview @@ -131,6 +133,7 @@ export function setupResolvers(): void { HtmlResolver.previewRenderer = ContentPreview ZipResolver.previewRenderer = ContentPreview TextResolver.solvedContentRenderer = ContentText + SourceResolver.solvedContentRenderer = ContentText AudioResolver.solvedContentRenderer = ContentAudio EpubResolver.solvedContentRenderer = ContentFile FileResolver.solvedContentRenderer = ContentFile diff --git a/apps/webext/README.md b/apps/webext/README.md index 604046ea..d9d44226 100644 --- a/apps/webext/README.md +++ b/apps/webext/README.md @@ -16,7 +16,7 @@ HTML 和文字是普通 core 语义内容块，PDF 使用当前 Peer 支持的 本机待发送与已保存到信息库是不同状态。待办绑定入队时的信息库与本 Peer 身份，后台自行推进，不靠 popup 或设置页轮询触发；关闭界面或重启后仍可继续。暂停和排除停止未派发的自动工作，手动保存仍可使用；排除编辑器回填现有规则，刷新状态不覆盖未保存的编辑。恢复尽力复用可见图结果，但不承诺 exactly-once、全图事务或完整历史。响应丢失、并发或中途退出可能留下重复或孤立项，失败不会被显示为完整保存。 -书签和历史先预览范围，确认后保存来源事实并尝试取得当前正文。记录时间与当前取得时间分别保留。登录、失效或不可读项保留来源和原因，支持停止、继续和失败重试；原书签删除不自动删除信息库资料。第三方收藏服务连接器仍待选定真实资料源。 +书签和历史先预览范围，确认后保存来源事实并尝试取得当前正文。记录时间与当前取得时间分别保留。登录、失效或不可读项保留来源和原因，支持停止、继续和失败重试；原书签删除不自动删除信息库资料。第三方收藏服务由独立插件提供，Web-ext 本身不保存站点选择器或收藏模型。 阅读页提供文本、出处、附注、HTML 源码和原始 PDF 下载，不执行来源脚本。不承诺取得尚未加载、虚拟化移除、canvas、封闭组件或不可读跨域 frame 的正文；已发现缺口会标注。 @@ -24,6 +24,17 @@ Explain、Writing Assist、旧选区浮栏、图编辑侧栏和模型设置已 升级时重新加载扩展，并刷新此前已注入旧浮栏的网页，让旧版本留在页面中的内容一并清除。 +## 让 Web Source 使用浏览器 + +Chromium 125 及以上版本可为 InKCre Web 中运行的 Source 提供浏览器能力。安装或升级这版扩展需要原生 debugger 权限；具体 Web 地址仍须由你单独授权。 + +1. 在扩展设置中配置当前信息库连接。 +2. 在“浏览器操作”输入可信任的 InKCre Web 地址，点击“授权此站点”并完成原生站点权限操作。 +3. 回到 Web 的业务插件设置，查找并明确绑定此浏览器，再选择要采集的范围。B 站等网站的登录直接在该浏览器中完成，无需向 Source 复制 cookie。 +4. 从 Web 运行普通采集任务。扩展负责真实标签页操作，业务插件负责解释和保存信息。 + +可在扩展设置撤销某个 Web 地址的授权，这会停止该授权下的浏览器会话。开始操作时会切换该浏览器窗口的活动标签页；采集新建的页面结束后关闭，借用的已有页面保留。Firefox 版本仍提供既有材料采集，暂不提供这条自动化路径。当前不承诺关闭 Web 或浏览器后继续运行采集任务。 + ## 开发与验证 使用根仓库声明的 Node.js/pnpm 版本，运行 `pnpm install --frozen-lockfile`。本目录的 `pnpm dev`、`pnpm build` 面向 Chromium，`pnpm dev:firefox`、`pnpm build:firefox` 面向 Firefox；`pnpm type-check` 检查扩展。 @@ -36,7 +47,7 @@ Explain、Writing Assist、旧选区浮栏、图编辑侧栏和模型设置已 WebExt 固定使用正式发布的 `@inkcre/ui-web` 2.6.0，通过内置密码显隐能力、公开 `INK_I18N_KEY` 和中文词典提供按钮名称，不另写密码组件。2.5.0 和 2.6.0 的普通可编辑输入框都会额外渲染只读值和重复 ID，因此仓库分别维护版本限定的 pnpm patch，各修正一个渲染条件；升级到上游修复版后移除对应补丁。这次仅 WebExt 升级，web、twitter 和 ext-dev-utils 继续使用原 2.5.0 及其补丁，不改变组件 API、样式或仍使用 2.3.0 的扩展。 -根目录 `pnpm test:e2e:webext` 真正加载扩展。指定 `INKCRE_CHROMIUM_BINARY=/Applications/Helium.app/Contents/MacOS/Helium` 使用 Helium 的隔离测试 profile，测试结束清理，不连接日常会话。真实数据旅程还需要可销毁数据库/PostgREST 的 `INKCRE_E2E_WEBEXT_PGREST` 与 `INKCRE_E2E_WEBEXT_JWT_SECRET`。测试副本只对合成 localhost 资料预授予权限；原生权限 UI 另行验证，生产 manifest 保持 optional 权限。 +根目录 `pnpm test:e2e:webext` 真正加载扩展。指定 `INKCRE_CHROMIUM_BINARY=/Applications/Helium.app/Contents/MacOS/Helium` 使用 Helium 的隔离测试 profile，测试结束清理，不连接日常会话。真实数据旅程还需要可销毁数据库/PostgREST 的 `INKCRE_E2E_WEBEXT_PGREST` 与 `INKCRE_E2E_WEBEXT_JWT_SECRET`。测试副本只对合成 localhost 资料预授予权限；原生权限 UI 另行验证，合成 host 的预授权不进入生产 manifest；Chromium 的 debugger 为必选权限，站点权限仍按需申请。 实现合同见[WebExt 运行时与材料](../../docs/30-unit-tdd/webext-collection.md)。安装、手动更新、旧身份迁移和 Changesets/GitHub Release 流程见[WebExt 版本与分发](../../docs/40-deployment/webext-delivery.md)。本地构建不等于发布；Firefox ZIP 尚未签名，不能作为正式安装包。 diff --git a/apps/webext/components/BrowserAutomationSettings.vue b/apps/webext/components/BrowserAutomationSettings.vue new file mode 100644 index 00000000..5df59e19 --- /dev/null +++ b/apps/webext/components/BrowserAutomationSettings.vue @@ -0,0 +1,88 @@ + + + diff --git a/apps/webext/entrypoints/background.ts b/apps/webext/entrypoints/background.ts index d860b024..1c4c3542 100644 --- a/apps/webext/entrypoints/background.ts +++ b/apps/webext/entrypoints/background.ts @@ -1,4 +1,5 @@ import { browser } from 'wxt/browser' +import { initializeBrowserAutomation } from '~/logic/browser-automation/runtime' import { handleCollectionMessage, handlesCollectionMessage, @@ -9,6 +10,7 @@ import { } from '~/logic/collection/runtime' export default defineBackground(() => { + initializeBrowserAutomation() const initialized = initializeCollection() initialized .then(() => Promise.all([wakeWork(), refreshPeers()])) diff --git a/apps/webext/entrypoints/browser-bridge.content.ts b/apps/webext/entrypoints/browser-bridge.content.ts new file mode 100644 index 00000000..035821a7 --- /dev/null +++ b/apps/webext/entrypoints/browser-bridge.content.ts @@ -0,0 +1,84 @@ +import { browser, type Browser } from 'wxt/browser' +import { + BROWSER_AUTOMATION_PORT, + BROWSER_AUTOMATION_PROTOCOL, + BrowserBridgeRequestSchema, + BrowserBridgeResponseSchema, + BrowserBridgeEventSchema, + type BrowserBridgeRequest, + type BrowserBridgeEvent, +} from '@inkcre/core/browser-automation/protocol' + +export default defineContentScript({ + matches: [], + registration: 'runtime', + main(ctx) { + // Granting a second information-base binding may inject the bridge again in the same page. + const marker = '__inkcreBrowserBridgeV1' + const scope = globalThis as typeof globalThis & { [marker]?: boolean } + if (scope[marker]) return + scope[marker] = true + let port: Browser.runtime.Port | undefined + const pending = new Map() + const sessions = new Map>() + const disconnected = () => { + for (const request of pending.values()) { + if (!request.provider || !request.instance) continue + window.postMessage( + { + ...request, + kind: 'response', + ok: false, + error: '浏览器桥已断开；已发送操作的结果可能未知。', + }, + location.origin + ) + } + for (const session of sessions.values()) + window.postMessage({ ...session, event: 'closed' }, location.origin) + pending.clear() + sessions.clear() + port = undefined + } + ctx.addEventListener(window, 'message', (event: MessageEvent) => { + if (event.source !== window || event.origin !== location.origin) return + const request = BrowserBridgeRequestSchema.safeParse(event.data) + if (!request.success) return + if (!port) { + port = browser.runtime.connect({ name: BROWSER_AUTOMATION_PORT }) + port.onMessage.addListener((reply: unknown) => { + const response = BrowserBridgeResponseSchema.safeParse(reply) + if (response.success) { + const outgoing = pending.get(response.data.request) + const opened = response.data.value as { session?: string } | undefined + if (outgoing?.operation === 'open' && response.data.ok && opened?.session) + sessions.set(opened.session, { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'event', + client: response.data.client, + provider: response.data.provider, + instance: response.data.instance, + session: opened.session, + }) + pending.delete(response.data.request) + } + const event = BrowserBridgeEventSchema.safeParse(reply) + if (event.success && event.data.event === 'closed') sessions.delete(event.data.session) + window.postMessage(reply, location.origin) + }) + port.onDisconnect.addListener(disconnected) + } + try { + pending.set(request.data.request, request.data) + port.postMessage(request.data) + } catch { + disconnected() + } + }) + ctx.addEventListener(window, 'pagehide', () => port?.disconnect()) + ctx.onInvalidated(() => { + port?.disconnect() + delete scope[marker] + }) + }, +}) diff --git a/apps/webext/entrypoints/options/Options.vue b/apps/webext/entrypoints/options/Options.vue index 20b48baf..9f4ec580 100644 --- a/apps/webext/entrypoints/options/Options.vue +++ b/apps/webext/entrypoints/options/Options.vue @@ -4,6 +4,7 @@ import { browser } from 'wxt/browser' import { InkButton, InkDropdown, InkForm, InkInput, InkSwitch, InkTextarea } from '@inkcre/ui-web' import type { MaterialRead } from '@inkcre/web-collection' import MaterialReader from '~/components/collection/MaterialReader.vue' +import BrowserAutomationSettings from '~/components/BrowserAutomationSettings.vue' import logo from '@inkcre/ui-web/logo/svg/original-transparent/inkcre-original-transparent-32.svg?url' import reverseLogo from '@inkcre/ui-web/logo/svg/reverse-transparent/inkcre-reverse-transparent-32.svg?url' import { collectionMessage } from '~/logic/collection/messages' @@ -202,6 +203,7 @@ onMounted(() => { +

信息库

crypto.randomUUID()), + grants: z.array(z.object({ origin: z.url(), provider: z.uuid() })).default([]), +}) +type State = z.infer +type Port = Browser.runtime.Port +type Session = { + id: string + provider: string + client: string + port: Port + tabId: number + created: boolean + transport: ExtensionTransport + detached: boolean + closing?: Promise +} +const epoch = crypto.randomUUID() +const sessions = new Map() +const occupiedTabs = new Set() +const ports = new Map() +const tabReferences = new Map() + +export async function automationState(): Promise { + const stored = await browser.storage.local.get(STORAGE_KEY) + const state = StateSchema.parse(stored[STORAGE_KEY] ?? {}) + if (!stored[STORAGE_KEY]) await browser.storage.local.set({ [STORAGE_KEY]: state }) + return state +} + +function originOf(value: string): string { + const url = new URL(value) + if (!['http:', 'https:'].includes(url.protocol)) throw new Error('只支持 HTTP(S) Web 地址。') + if (url.username || url.password) throw new Error('Web 地址不能包含凭据。') + return url.origin +} + +export async function grantAutomation(origin: string, provider: string): Promise { + if (!import.meta.env.CHROME) throw new Error('此浏览器不支持调试自动化；请使用 Chromium。') + const exactOrigin = originOf(origin) + if (!(await connections()).some((target) => target.peerId === provider)) + throw new Error('请先为此浏览器配置所选信息库连接。') + if (!(await browser.permissions.contains({ origins: [siteMatchPattern(exactOrigin)] }))) + throw new Error('请先授权此 Web 站点。') + const state = await automationState() + state.grants = [ + ...state.grants.filter((grant) => grant.origin !== exactOrigin || grant.provider !== provider), + { origin: exactOrigin, provider }, + ] + await browser.storage.local.set({ [STORAGE_KEY]: state }) + await syncAutomationScripts() + for (const tab of await browser.tabs.query({})) { + if (tab.id === undefined || !tab.url || new URL(tab.url).origin !== exactOrigin) continue + await browser.scripting + .executeScript({ target: { tabId: tab.id }, files: ['/content-scripts/browser-bridge.js'] }) + .catch(() => undefined) + } +} + +export async function revokeAutomation(origin: string, provider: string): Promise { + const state = await automationState() + state.grants = state.grants.filter( + (grant) => grant.origin !== origin || grant.provider !== provider + ) + await browser.storage.local.set({ [STORAGE_KEY]: state }) + await Promise.all( + [...sessions.values()] + .filter((session) => ports.get(session.port) === origin && session.provider === provider) + .map((session) => release(session)) + ) + if (!state.grants.some((grant) => grant.origin === origin)) + for (const [port, portOrigin] of ports) if (portOrigin === origin) port.disconnect() + await syncAutomationScripts() +} + +export async function syncAutomationScripts(): Promise { + const registered = (await browser.scripting.getRegisteredContentScripts()) + .filter((script) => script.id.startsWith('inkcre-browser-')) + .map((script) => script.id) + if (registered.length) await browser.scripting.unregisterContentScripts({ ids: registered }) + if (!import.meta.env.CHROME) return + const state = await automationState() + const matches: string[] = [] + for (const grant of state.grants) { + const pattern = siteMatchPattern(grant.origin) + if (!matches.includes(pattern) && (await browser.permissions.contains({ origins: [pattern] }))) + matches.push(pattern) + } + if (!matches.length) return + await browser.scripting.registerContentScripts([ + { + id: 'inkcre-browser-bridge-v1', + matches, + js: ['content-scripts/browser-bridge.js'], + runAt: 'document_start', + allFrames: false, + persistAcrossSessions: true, + }, + ]) +} + +function post(port: Port, value: unknown): void { + try { + port.postMessage(value) + } catch { + // A disappearing Web page is handled by the Port's disconnect cleanup. + } +} + +function release(session: Session): Promise { + if (session.closing) return session.closing + if (!sessions.has(session.id)) return Promise.resolve() + session.closing = (async () => { + try { + await session.transport.close() + } catch (error) { + if (!session.detached) throw error + } finally { + if (session.created) await browser.tabs.remove(session.tabId).catch(() => undefined) + sessions.delete(session.id) + occupiedTabs.delete(session.tabId) + const state = await automationState() + post(session.port, { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'event', + client: session.client, + provider: session.provider, + instance: state.instance, + session: session.id, + event: 'closed', + }) + } + })() + return session.closing +} + +async function providers(origin: string): Promise { + const state = await automationState() + const targets = await connections() + const allowed = state.grants.filter((grant) => grant.origin === origin) + if (!(await browser.permissions.contains({ origins: [siteMatchPattern(origin)] }))) return [] + return targets + .filter((target) => allowed.some((grant) => grant.provider === target.peerId)) + .map((target) => ({ + provider: target.peerId, + instance: state.instance, + name: target.name, + extensionVersion: browser.runtime.getManifest().version, + protocol: 1, + })) +} + +function tabRef(tab: Browser.tabs.Tab, binding: BrowserProviderBinding): BrowserTabRef { + if (tab.id === undefined) throw new Error('浏览器未返回标签页身份。') + return { + provider: binding.provider, + instance: binding.instance, + epoch, + reference: crypto.randomUUID(), + tabId: tab.id, + title: tab.title, + url: tab.url, + } +} + +async function dispatch( + port: Port, + request: BrowserBridgeRequest, + binding: BrowserProviderBinding +): Promise { + if (request.operation === 'tabs') { + for (const [reference, issued] of tabReferences) + if (issued.port === port && issued.client === request.client) tabReferences.delete(reference) + const tabs = (await browser.tabs.query({})) + .filter((tab) => tab.id !== port.sender?.tab?.id && tab.url && /^https?:\/\//.test(tab.url)) + .map((tab) => tabRef(tab, binding)) + for (const tab of tabs) tabReferences.set(tab.reference, { tab, port, client: request.client }) + return tabs + } + + if (request.operation === 'open') { + const parameters = z + .object({ + session: z.uuid(), + target: z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('new') }), + z.object({ kind: z.literal('existing'), tab: BrowserTabRefSchema }), + ]), + }) + .parse(request.parameters) + if (sessions.has(parameters.session)) throw new Error('浏览器会话身份已被使用。') + const created = parameters.target.kind === 'new' + if (parameters.target.kind === 'existing') { + const ref = parameters.target.tab + if ( + ref.provider !== binding.provider || + ref.instance !== binding.instance || + ref.epoch !== epoch + ) + throw new Error('标签页引用已过期或属于其他浏览器。') + const issued = tabReferences.get(ref.reference) + if ( + !issued || + issued.port !== port || + issued.client !== request.client || + issued.tab.tabId !== ref.tabId + ) + throw new Error('标签页引用已经使用或过期，请刷新标签页列表。') + if (ref.tabId === port.sender?.tab?.id) + throw new Error('不能控制正在运行此 Source 的 Web 标签页。') + if (occupiedTabs.has(ref.tabId)) throw new Error('此标签页正在被另一个自动化会话使用。') + for (const [reference, issued] of tabReferences) + if (issued.tab.tabId === ref.tabId) tabReferences.delete(reference) + occupiedTabs.add(ref.tabId) + } + let tab: Browser.tabs.Tab | undefined + let transport: ExtensionTransport | undefined + try { + tab = created + ? await browser.tabs.create({ url: 'about:blank', active: true }) + : await browser.tabs.update( + parameters.target.kind === 'existing' ? parameters.target.tab.tabId : -1, + { active: true } + ) + if (!tab || tab.id === undefined) throw new Error('无法取得浏览器标签页。') + occupiedTabs.add(tab.id) + transport = await ExtensionTransport.connectTab(tab.id) + const session: Session = { + id: parameters.session, + provider: binding.provider, + client: request.client, + port, + tabId: tab.id, + created, + transport, + detached: false, + } + sessions.set(session.id, session) + transport.onmessage = (data) => + post(port, { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'event', + client: request.client, + provider: binding.provider, + instance: binding.instance, + session: session.id, + event: 'message', + data, + }) + transport.onclose = () => void release(session) + if (!ports.has(port)) { + await release(session) + throw new Error('调用页面已断开，浏览器会话已释放。') + } + return { session: session.id, tab: tabRef(tab, binding) } + } catch (error) { + if (tab?.id !== undefined) { + occupiedTabs.delete(tab.id) + await transport?.close() + if (created) await browser.tabs.remove(tab.id).catch(() => undefined) + } else if (parameters.target.kind === 'existing') + occupiedTabs.delete(parameters.target.tab.tabId) + throw error + } + } + + const sessionId = z.uuid().parse(request.parameters.session) + const session = sessions.get(sessionId) + if (!session) { + if (request.operation === 'close') return null + throw new Error('浏览器会话已结束，旧引用不能再次执行。') + } + if ( + session.port !== port || + session.client !== request.client || + session.provider !== binding.provider + ) + throw new Error('此浏览器会话属于另一个调用者。') + if (request.operation === 'close') { + await release(session) + return null + } + if (request.operation === 'cdp') { + if (session.closing) throw new Error('浏览器会话正在释放。') + session.transport.send(z.string().parse(request.parameters.message)) + return null + } + throw new Error('不支持的浏览器操作。') +} + +async function handle(port: Port, origin: string, request: BrowserBridgeRequest): Promise { + const available = await providers(origin) + if (request.operation === 'discover') { + for (const binding of available) + post(port, { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'response', + client: request.client, + request: request.request, + provider: binding.provider, + instance: binding.instance, + ok: true, + value: binding, + }) + return + } + const binding = available.find( + (item) => item.provider === request.provider && item.instance === request.instance + ) + if (!binding) return + try { + if (!import.meta.env.CHROME) throw new Error('此浏览器不支持调试自动化。') + const value = await dispatch(port, request, binding) + post(port, { + ...request, + kind: 'response', + provider: binding.provider, + instance: binding.instance, + ok: true, + value, + }) + } catch (error) { + post(port, { + ...request, + kind: 'response', + provider: binding.provider, + instance: binding.instance, + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } +} + +export function initializeBrowserAutomation(): void { + void syncAutomationScripts().catch((error) => + console.error('Browser bridge initialization failed:', error) + ) + browser.runtime.onMessage.addListener((raw: unknown, sender) => { + if (!raw || typeof raw !== 'object' || !('type' in raw)) return + if (!String(raw.type).startsWith('browser-automation-')) return + if (sender.id !== browser.runtime.id || !sender.url?.startsWith(browser.runtime.getURL('/'))) + return Promise.resolve({ ok: false, error: '仅扩展设置页面可修改浏览器授权。' }) + const message = raw as { type: string; origin?: string; provider?: string } + const execute = async () => { + if (message.type === 'browser-automation-grant') + await grantAutomation(z.string().parse(message.origin), z.uuid().parse(message.provider)) + else if (message.type === 'browser-automation-revoke') + await revokeAutomation(z.string().parse(message.origin), z.uuid().parse(message.provider)) + else if (message.type !== 'browser-automation-state') throw new Error('未知浏览器设置操作。') + return automationState() + } + return execute().then( + (value) => ({ ok: true, value }), + (error: unknown) => ({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + ) + }) + browser.runtime.onConnect.addListener((port) => { + const sender = port.sender + if ( + port.name !== BROWSER_AUTOMATION_PORT || + sender?.id !== browser.runtime.id || + sender.frameId !== 0 || + sender.tab?.id === undefined || + !sender.url + ) + return + let origin: string + try { + origin = originOf(sender.url) + } catch { + port.disconnect() + return + } + ports.set(port, origin) + // Opening and closing are serialized per page; CDP traffic must keep flowing concurrently. + let lifecycle = Promise.resolve() + port.onMessage.addListener((raw: unknown) => { + const parsed = BrowserBridgeRequestSchema.safeParse(raw) + if (!parsed.success) return + const run = () => handle(port, origin, parsed.data) + if (parsed.data.operation === 'open' || parsed.data.operation === 'close') { + lifecycle = lifecycle.then(run).catch(() => undefined) + } else void run().catch(() => undefined) + }) + port.onDisconnect.addListener(() => { + ports.delete(port) + for (const [reference, issued] of tabReferences) + if (issued.port === port) tabReferences.delete(reference) + for (const session of sessions.values()) if (session.port === port) void release(session) + }) + }) + if (import.meta.env.CHROME) + browser.debugger.onDetach.addListener((source) => { + for (const session of sessions.values()) { + if (session.tabId !== source.tabId) continue + session.detached = true + void release(session).catch(() => undefined) + } + }) + browser.permissions.onRemoved.addListener(() => { + void syncAutomationScripts() + for (const session of sessions.values()) { + const origin = ports.get(session.port) + if (origin) + void providers(origin).then((available) => { + if (!available.some((binding) => binding.provider === session.provider)) + void release(session) + }) + } + }) +} diff --git a/apps/webext/package.json b/apps/webext/package.json index b0cd654d..59405b23 100644 --- a/apps/webext/package.json +++ b/apps/webext/package.json @@ -20,6 +20,7 @@ "@inkcre/ui-web": "2.7.0", "@inkcre/web-collection": "workspace:*", "@mozilla/readability": "^0.6.0", + "puppeteer-core": "25.16.0", "vue": "^3.5.44", "zod": "^4.6.7" }, diff --git a/apps/webext/wxt.config.ts b/apps/webext/wxt.config.ts index 3f016521..7cb5451b 100644 --- a/apps/webext/wxt.config.ts +++ b/apps/webext/wxt.config.ts @@ -74,11 +74,11 @@ export default defineConfig({ 'activeTab', 'scripting', 'alarms', - ...(browser === 'firefox' ? [] : ['offscreen']), + ...(browser === 'firefox' ? [] : ['offscreen', 'debugger']), ], optional_permissions: ['bookmarks', 'history'], optional_host_permissions: ['http://*/*', 'https://*/*'], - ...(browser === 'firefox' ? {} : { minimum_chrome_version: '116' }), + ...(browser === 'firefox' ? {} : { minimum_chrome_version: '125' }), ...(browser === 'firefox' ? { browser_specific_settings: { @@ -105,6 +105,9 @@ export default defineConfig({ vite: () => ({ resolve: { alias: { + '@inkcre/core/browser-automation/protocol': fileURLToPath( + new URL('../../packages/core/src/browser-automation/protocol.ts', import.meta.url) + ), '@inkcre/core': fileURLToPath(new URL('../../packages/core/src/index.ts', import.meta.url)), }, }, diff --git a/docs/30-unit-tdd/README.md b/docs/30-unit-tdd/README.md index d610f656..8f54b76e 100644 --- a/docs/30-unit-tdd/README.md +++ b/docs/30-unit-tdd/README.md @@ -13,6 +13,8 @@ not architectural boundaries. browser runtime handles, and rendering boundaries. - [Native Extension Runtime](native-extension-runtime.md) owns the Extension Host, producer/Host boundary, native Module Federation loading, durable enabled intent, and compensated lifecycle. +- [Web Source 本机浏览器能力](browser-automation.md) owns origin pairing, exact browser binding, + Puppeteer transport, and the lifetime of borrowed or created automation tabs. Repository/package topology remains in [`../../ARCHITECTURE.md`](../../ARCHITECTURE.md). Observable product behavior and cross-unit contracts remain in the read-only diff --git a/docs/30-unit-tdd/bilibili-extension.md b/docs/30-unit-tdd/bilibili-extension.md new file mode 100644 index 00000000..9c5680af --- /dev/null +++ b/docs/30-unit-tdd/bilibili-extension.md @@ -0,0 +1,37 @@ +# B 站收藏夹插件 + +本文适用于 `inkcre/bilibili` Web MF Distribution 的维护者。共同产品承诺与跨 Distribution 合同由共享文档拥有；本 Unit 负责浏览器站点适配、Source 执行、图投影及读取界面。Python 只读 Distribution 解释相同 v1 记录，为既有读取和词法检索提供能力，不拥有或控制 Web Source 的执行。 + +## 所有权 + +插件拥有 B 站账号、收藏夹、视频和收藏条目的解释，站点路径与接口字段也留在插件内。通用 Core / Web-ext 只提供浏览器绑定、Puppeteer Page、消息桥和会话生命周期，不包含 B 站选择器或业务模型。Web Peer 以普通 `core.source.collect.v1` Job 执行 Source。 + +首版范围为用户自己创建且明确选定的收藏夹，含本人可访问的私密收藏夹。账号、收藏夹选择和浏览器绑定写入 Source 配置；浏览器 cookie、密码和视频二进制不会写入配置或业务图。第三方站点接口不是 InKCre 对外稳定合同，其响应由 `site.ts` / `normalize.ts` 转换。 + +## 共享图的本地实现 + +四个精确 Resolver 为 `extensions.bilibili.user.v1`、`folder.v1`、`video.v1`、`membership.v1`，完整前缀相同。领域内容校验由 `schema.ts` 执行。账号以 MID、收藏夹以完整 media_id、视频以 aid 标识；BVID 是已知视频别名，原始 fid 不能代替 media_id。 + +账号到收藏夹使用 `owns`，收藏夹到成员使用 `contains`，成员到视频使用 `item`，视频到 UP 主使用 `author`。成员自己的 Block 承载可变收藏时间、观察位置与条目可用性；稳定边不把这些值串入身份。观察位置明确为 `page_number` 与 `position_in_page`，不由分页大小推算并声称原生绝对顺序。内容不重复保存跨实体关联作为第二份权威。 + +扩展通过普通 Block / Relation API 自行核对原生对象，每次执行读取已有 B 站内联 JSON 块，以实际解析出的 MID、完整 media_id 或 aid 复用 Block。查询按 Resolver 和普通分页限定范围，JSON 空格与属性顺序不参与身份判断。成员身份来自明确的收藏夹 `contains` 成员 `item` 视频路径；v1 成员内容不增加 folder_id、aid 或私有 native_key。执行中的内存查找表不作为共享权威，也不保存第二份持久身份账本。 + +来源没有取得的字段不写入 patch，扩展在已有 JSON 对象上合并已知字段。失效视频的占位标题、空缺作者或简介不能抹掉已保存事实。独立视频身份不依赖某次 Source 或浏览器，重复采集和跨收藏夹采集复用相同 Block；本次采集只补齐自己的稳定边，不删除用户关系。 + +## 批次与恢复 + +`collect.ts` 按配置的收藏夹逐一读取，每批最多处理一个 20 项分页；`has_more` 是结束依据。来源可能返回不足 20 项但仍有后续，不能用条目数代替结束标记。 + +Source.state 仅保存范围、分页提示、首批和已提交边界的身份序列、进度及计数，不存平行信息内容。继续前比较首批身份、来源报告数量及已提交边界；不同则从头重新核对该收藏夹。此策略减少普通新增和移动造成的漏取，不提供实时列表的原子快照或自动删除语义。 + +末批提交前重新读取首批，对照本轮首批身份、起始报告数量与末批报告数量；发生变化时不提交“完成”，保留此前已确认的批次，并使 Job 明确失败。完整遍历与条目全数取得是不同声明：每个收藏夹同时保留累计返回、解析视频、失效、未支持、未解析数量和来源报告数，设置界面展示差额。数量不等本身不会触发无限重扫，也不能被隐去后声称完整；此边界检查不检测任意中间位置变化。 + +每页先确认对象和关系写入，最后以普通 Source 更新保存进度。失败停止，创建结果未知时不在同一次执行中重放。下一次执行重新读取普通图：已可确认身份的对象和完整成员路径会被复用，缺失的边会被补齐；在成员身份路径尚未形成前中断可能留下孤立块，不为消除此残留增加持久账本。本版不保证并发首次创建唯一或整页原子可见。 + +`Source.block` 指向该 Source 独占的 `core.source.v1` 锚点，锚点以稳定 `collects` 边指向共享账号业务根。没有锚点引用时先按普通图中的 Source id 查找，避免因一次来源锚点创建响应丢失就直接重复创建；锚点和业务账号不能互换。最后更新 Source 的锚点引用与进度。不同 Source 可采集同一个账号而保持各自来源身份。设置变更只更新配置，不顺便写业务图。 + +## 读取 + +Resolver 仅依赖已保存的 Block 和稳定关系；不会为读取而连接原浏览器或调用 B 站。收藏条目按归档关系分批读取，每次最多 50 个并提供继续入口，避免大收藏夹在单个图预览中被截断。视频所在收藏夹经成员 Block 反向解析，收藏时间与可用性跟随对应成员显示。 + +首轮仅保存元数据。原文和封面 URL 仍指向来源；文字和关系的离线使用不取决于这些链接继续可用。视频播放、字幕、源站编辑与失效内容恢复不在此 Unit 的当前承诺中。 diff --git a/docs/30-unit-tdd/browser-automation.md b/docs/30-unit-tdd/browser-automation.md new file mode 100644 index 00000000..5e940b04 --- /dev/null +++ b/docs/30-unit-tdd/browser-automation.md @@ -0,0 +1,63 @@ +# Web Source 的本机浏览器能力 + +当 Web 中运行的 extension Source 需要借用用户浏览器的登录态、页面交互或页面网络响应时，使用本 Unit。Source 继续由普通 MF Distribution 交付，Web Peer 通过共享 Job 协议领取任务；Web-ext 承担本机标签页和调试会话执行。站点业务逻辑与数据模型不进入 SDK 或 Web-ext。 + +## 资格与配置 + +`SourceImplementation.canCollect(source, config)` 与 `canBackfill` 是可选的业务资格钩子。普通 Source 没有钩子时沿用已注册实现的判断。浏览器 Source 可调用 `BrowserAccess.hasBinding(provider)` 检查本 Web Peer 是否已明确配置该兼容 Provider；不得用实时连接、权限或登录检查使任务无声停留 pending。命令配置不合法时仍进入执行验证并失败。 + +Provider 是 Web-ext 加入所选信息库时配置的确切 Peer UUID。浏览器实例另有保存在扩展本机 storage 的随机 `instance`；运行中的 service worker 再提供短期 `epoch`。绑定同时固定 Provider 和 instance，不根据另一个在线实例替换浏览器或账号。 + +Web 绑定保存在本机 localStorage，以当前 Web Peer UUID 和数据库地址分隔。`bindings()` 和 `hasBinding()` 只读这份配置；它们不声称浏览器当前在线。未配置时 Source 不领取；已有绑定但桥断开、授权撤销或账号失效时，已领取的执行明确失败。 + +## 原生授权和通信 + +1. 用户在 Web-ext 设置中为当前信息库授权可信的 InKCre Web 地址。 +2. 原生 host 权限通过明确的用户操作请求，扩展只为授权的站点注册打包内的 bridge content script。 +3. Web 调用 `BrowserAccess.discover()`，用户明确选择后 `bind(binding)`；选择结果不是自动授予扩展端权限。 +4. Web 页面通过同 origin 的 `window.postMessage` 与 content script 通信，再由 `runtime.Port` 到 background。background 对每次请求检查真实 sender、顶层 frame、完整 origin、当前原生授权及精确 Provider/instance。 + +浏览器的 host 权限模式不区分端口；扩展端的完整 origin 授权会区分。允许一个 localhost 端口不会使另一端口获得自动化权限。消息的名称、UUID 和 namespace 都不替代授权；同一已授权 origin 的页面脚本处于同一个信任范围。 + +动态授权适用于自托管地址，不需要将用户域名写入 manifest，也不需要云端或 HTTP 中继。撤销授权释放该 origin 的会话并移除后续脚本注册。已有页面上的 bridge 不能绕过 background 的实时检查。 + +## Source 接口 + +```ts +await BrowserAccess.withPage( + { provider: configuredProvider, target: { kind: 'new' }, signal }, + async (page) => { + await page.goto(sourceUrl) + await page.waitForSelector('[data-ready]') + return page.evaluate(() => document.title) + } +) +``` + +`BrowserPage` 是 Puppeteer 的真实 `Page` 类型。`tabs(provider, signal?)` 返回当前可借用的 HTTP(S) 标签页短期引用；`target: {kind: 'existing', tab}` 必须使用当前枚举结果。引用绑定调用页面、Provider、实例、service-worker epoch 和单次 reference UUID，枚举刷新或一次使用后失效；不能把裸 tabId 当成可长期持有的权限。 + +新建标签页正常结束时关闭；借用标签页解除调试并保留。开始控制时将目标设为其浏览器窗口的活动标签页，保证浏览器渲染和原生点击可执行；这会改变该窗口的活动标签页，不声称恢复用户原来的浏览位置。相同标签页的控制互斥，旧 Page 不能在结束后继续发命令。网页已发生的变化不会回滚，Source 自己负责账号确认、站点身份、分页、预算、增量进度和业务写入。 + +Abort 会中断 Page 连接、请求后台释放，并等待 Source callback 退出与原生清理；不会通过 Promise.race 提前把仍执行中的 callback 算作结束。bridge 断连向请求与 Page 传播失败。已经派发的效果可能未知，不自动重放命令或替换 Provider。打开动作的会话 ID 由调用方预先产生，打开响应丢失后仍能向原目标请求清理。 + +## Puppeteer 适配边界 + +依赖固定为 `puppeteer-core@25.16.0`，两端使用同一版本。Web-ext 复用官方实验性 `ExtensionTransport`，由 `chrome.debugger` 执行；每个连接对应一个标签页，标签页创建和释放由 Web-ext 完成。 + +25.16 的 public browser entry 在构建图中仍引入 Node-only 的 channel 连接分支。SDK 使用该固定版本的窄 CDP connector，等同于官方 connect 在给定 transport 后选择的 CDP 路径，避免要求每个 Host/MF producer 设置 Node externals。它是明确的内部入口维护点；升级依赖时需重新检查与实测。Page、Connection 和自动化命令仍来自 Puppeteer，不是自实现的兼容层。background 静态导入窄 ExtensionTransport 模块，避免 service worker 执行动态 import。 + +该版本的 `ExtensionTransport.close()` 原本丢弃 native detach Promise。本仓库仅补丁修改其返回值及声明，使外层可以等待原生 detach 完成；补丁行为测试验证等待和事件监听清理。外层另监听原生 `debugger.onDetach`，处理目标关闭、DevTools 抢占等会话中断。上游提供等价保证时移除该补丁。 + +Chromium 构建声明必选 `debugger` 权限，最低 Chrome 125，覆盖 flat child sessions；该权限不能作为 optional 权限运行时申请。Firefox 构建保留既有材料采集，但不暴露这条 debugger 自动化能力。不能从一次 Page 操作通过推定完整 Puppeteer API 兼容，也不承诺关闭 Web 或浏览器后继续 Job。 + +协议 DTO 由 `@inkcre/core/browser-automation/protocol` 独立入口交付，content script 只导入这个轻量入口，避免运行 Core root 的 Job 注册等副作用。 + +## 验证入口 + +`pnpm exec node --test scripts/tests/browser-automation-transport.test.mjs` 检查限定版本补丁的 native Promise 行为。`pnpm exec node scripts/verify-browser-automation.mjs` 使用生产扩展构建、真实 SDK 和隔离 Chromium profile 运行合成登录站点；测试进程不连接远程 CDP，因此不会占用被测标签页的调试连接。它检查实际导航、登录态、点击和后续批次、响应体、标签页清理、借用页保留、旧引用、Abort、精确 Provider 与同 host 不同 origin 的授权边界。这些检查也接入 `pnpm test:e2e:webext`，位于既有材料采集回归之前。 + +同 origin 的多个 Provider 共用 Port。撤销 A 只释放 A 的会话；B 仍获授权时不能断开共用 Port。隔离验收在 B 的真实 Page 会话活跃时撤销 A，检查 B 仍可执行。`scripts/tests/job-cancellation.test.mjs` 检查已取消 pending Job 及 claim 期间收到取消时不启动业务 handler；普通不合格任务继续 pending。 + +合成站点的测试副本只为合成 host 预授予权限，不能替代原生授权 UI 或真实 B 站登录态的业务验收。用户实际业务数据的验收由相应扩展与任务证据维护。 + +依据：[Puppeteer 扩展指南](https://pptr.dev/guides/running-puppeteer-in-extensions)、[Chrome debugger](https://developer.chrome.com/docs/extensions/reference/api/debugger)。 diff --git a/docs/30-unit-tdd/info-base.md b/docs/30-unit-tdd/info-base.md index 3c923190..981fb1fa 100644 --- a/docs/30-unit-tdd/info-base.md +++ b/docs/30-unit-tdd/info-base.md @@ -75,6 +75,12 @@ manager does not impose a universal classification ladder. ## Browser Handles and Rendering +`core.source.v1` is a provenance anchor rather than a business collection root. Its strict content +contains only Source `id`, `type`, and optional `nickname`; the native Source Resolver projects the +same readable sentence as the Python implementation and uses the existing text/preview renderers. +Opening a Source does not load credentials or run a collector. Outgoing domain relations lead to +the persisted business graph, which keeps its own exact Resolver contracts. + Byte-oriented Resolvers may expose Blob or Object URL handles for image, audio, video, PDF, EPUB, ZIP, and file content. Handles are private runtime state, never persisted solved-graph authority. They are revoked on Resolver refresh, disposal, cache replacement, and eviction. diff --git a/docs/40-deployment/native-extension-delivery.md b/docs/40-deployment/native-extension-delivery.md index 248021b1..cb01a86e 100644 --- a/docs/40-deployment/native-extension-delivery.md +++ b/docs/40-deployment/native-extension-delivery.md @@ -71,11 +71,12 @@ copy so the preview can load a mutually compatible Host and MF when a change rai Host SDK. This produces temporary derived versions for preview only. It neither consumes the pending changesets in the PR nor replaces the normal Version PR and protected-main release lifecycle. -The Workspace contract check applies the same preparation after `pnpm check` has validated the -original checkout. It builds the isolated candidate Host and MF before inspecting the native -snapshot, and records the exact checked-out commit SHA in the job summary. For pull requests that -SHA can be GitHub's synthetic merge commit; it identifies the source actually exported by -`git archive`, while static, typed, and package checks still run on the original checkout first. +Workspace contract 先校验源代码格式、静态规则和类型，再读取 Changesets 的版本计划。有待升版的包时， +在 CI 的临时 checkout 中运行 `pnpm release:version` 与 `pnpm install --lockfile-only --ignore-scripts`，然后构建并验证 Host 和 MF。 +Pages preview 对受信控制器选定的 PR head 使用同样的版本计划和命令，保留完整历史供 Changesets 读取。 +这些派生版本仅用于验证和预览，不回写源分支，也不替代 Version PR 或正式发布。 + +本地需要验证尚未发布的 SDK 时，在临时 clone 中运行上述命令及 `pnpm build`；日常工作树继续保留版本意图。 A plugin owns compatibility of its shared configuration, state, Source/Job parameters and stored information. Changesets record MF changes even when no Python change is necessary. Published native diff --git a/docs/40-deployment/web-delivery.md b/docs/40-deployment/web-delivery.md index 198356e2..6426a949 100644 --- a/docs/40-deployment/web-delivery.md +++ b/docs/40-deployment/web-delivery.md @@ -46,9 +46,10 @@ into chunks and fetched on first navigation; repeat visits can reuse unchanged b [`.github/workflows/pages-deploy.yml`](../../.github/workflows/pages-deploy.yml) owns production delivery. A protected-`main` push selects that exact source SHA after strict required PR checks, while delivery rejects the run if a newer revision supersedes it. The delivery job checks out the -exact source, installs the frozen workspace, builds the release, reverifies `main`, and deploys -those same-run bytes to the Cloudflare Pages `main` branch in the protected `production` -environment. +exact source, installs the frozen workspace, and builds the Web App and its workspace dependencies. +It runs `pnpm check:release:web` before deploying those same-run bytes to the Cloudflare Pages +`main` branch in the protected `production` environment. Workspace CI checks Module Federation +compatibility, and first-party MF publication validates each selected Extension release. The workspace lock owns the exact Wrangler version, and Pages production/preview execute it through the pinned pnpm toolchain. Preview cleanup is intentionally different: its tombstone is a standalone diff --git a/docs/_shared b/docs/_shared index 17fc19e7..ad76242b 160000 --- a/docs/_shared +++ b/docs/_shared @@ -1 +1 @@ -Subproject commit 17fc19e7230187a7a989815e7b8303bbb380f478 +Subproject commit ad76242bd3a10536001aa751c1152bec7fb42990 diff --git a/extensions/bilibili/CHANGELOG.md b/extensions/bilibili/CHANGELOG.md new file mode 100644 index 00000000..2392e92b --- /dev/null +++ b/extensions/bilibili/CHANGELOG.md @@ -0,0 +1,7 @@ +# Bilibili Extension + +## 0.1.0 + +- 新增自己的公开及私密收藏夹采集、原生身份与成员关系存储，以及独立读取视图。 +- 通过所选浏览器的登录态执行普通 collection Job，支持有界分页和经边界复核的继续采集。 +- 保存文字与元数据，不保存视频二进制。 diff --git a/extensions/bilibili/LICENSE b/extensions/bilibili/LICENSE new file mode 100644 index 00000000..94e962af --- /dev/null +++ b/extensions/bilibili/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 InKCre contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/extensions/bilibili/README.md b/extensions/bilibili/README.md new file mode 100644 index 00000000..a068f21f --- /dev/null +++ b/extensions/bilibili/README.md @@ -0,0 +1,44 @@ +# B 站收藏夹 + +`inkcre/bilibili` 的 Module Federation Distribution，为 Web Peer 提供自己的 B 站收藏夹采集与读取能力。采集通过用户明确选择的浏览器执行，沿用其中的 B 站登录态；资料保存到共享信息库后，其他加载此插件的 Web Peer 可以独立读取。 + +## 使用 + +1. 在 InKCre 浏览器扩展设置中授权当前 Web 地址，并在同一浏览器登录 B 站。 +2. 打开插件设置，查找并选择该浏览器，读取 B 站收藏夹。 +3. 确认显示的账号，选择一个或多个自己的收藏夹并保存采集源。 +4. 开始采集。任务使用普通 Source / Job 路径，由具备此 Source 实现和对应浏览器绑定的 Web Peer 领取。 +5. 查看采集状态和已保存资料。单次分页预算用完时，任务可以正常结束，但设置会明确显示仍有未完成范围；再次执行会校验边界后继续。 + +MF Distribution 提供采集与独立读取能力。Python 的只读 Distribution 解释相同 v1 图合同，供既有读取与词法检索使用；它不是采集控制者，采集不要求运行 Python 业务进程。设置、账号确认或查找浏览器不会自动创建采集任务。运行期间应保持 Web Peer 与所选浏览器在线。任务运行时可以请求取消，界面会等待实际终态，不将请求发出等同于已经停止。 + +## 保存范围 + +- 自己创建且明确选择的收藏夹，包括私密收藏夹。 +- 收藏夹、视频、UP 主的原生身份和实际取得的标题、简介、链接等资料。 +- 收藏关系，以及来源明确返回的收藏时间、位置和条目可用性。同一视频位于多个收藏夹时共用视频实体。 +- 多 P 视频仍以被收藏的视频对象为单位。 + +本版不保存视频二进制，不下载字幕或评论，也不修改 B 站收藏。封面仅保存来源 URL；来源图片失效不会影响已保存的文字与关系。私密收藏夹的资料同样进入当前信息库，请按自己的信息库访问范围选择。 + +## 完整性与重复执行 + +分页以来源的 `has_more` 为准，短页不能被当成末尾。恢复前复核列表首批、数量和已提交分页边界；观察到变化时重新扫描。末批提交前再次核对首批和报告数量，发现变化则保留已确认批次并明确失败，供下一次重新核对。页码只是可复核的执行提示，不是永久信息身份。 + +“分页已遍历”仅表示可见分页到达结束标记并通过上述边界检查。界面同时显示累计返回条目、来源报告数量和差额；来源可能隐藏部分条目或异步更新计数，差额不能被当作已成功取得。列表在采集期间变化时，本版不承诺某一时刻的原子快照，也不声称能检测任意中间位置的重排。 + +采集扩展通过普通 Block / Relation API 读取并核对已有的原生对象，复用 Block ID、合并已取得的字段，并保留用户添加的关系。每页图写入全部确认后，才更新 Source 进度；中途失败会停止，已经写入的部分仍可能可见，下一次执行重新读取普通图并补齐。本版不承诺整页原子可见或并发首次创建唯一，也不会根据未出现的条目删除历史收藏资料。 + +创建结果未知时不会自动重放。若收藏成员在关联尚未连齐时中断，可能留下无法确认原生身份的孤立块；下一次只按明确的收藏夹—成员—视频路径复用成员，不猜测或误认其他条目。 + +失效占位标题不会覆盖已有标题或简介。非视频和身份不足条目计入未支持或未解析数量。未登录、账号变化、挑战响应、分页异常或断连会使任务明确失败，不能被当成空收藏夹成功。 + +## 开发与验证 + +```sh +pnpm --filter @inkcre/ext-bilibili test +pnpm --filter @inkcre/ext-bilibili type-check +pnpm --filter @inkcre/ext-bilibili build +``` + +单元回归使用虚构身份和文本，只复用实际观察到的响应字段形状；不在仓库保存用户私人收藏响应。真实浏览器链和私密收藏夹验收由主任务记录单独报告。 diff --git a/extensions/bilibili/index.html b/extensions/bilibili/index.html new file mode 100644 index 00000000..423310ef --- /dev/null +++ b/extensions/bilibili/index.html @@ -0,0 +1,12 @@ + + + + + + Bilibili Extension + + +
+ + + diff --git a/extensions/bilibili/package.json b/extensions/bilibili/package.json new file mode 100644 index 00000000..f8db12ca --- /dev/null +++ b/extensions/bilibili/package.json @@ -0,0 +1,48 @@ +{ + "name": "@inkcre/ext-bilibili", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "type-check": "vue-tsc --noEmit", + "test": "node --experimental-strip-types --test tests/*.test.ts" + }, + "dependencies": { + "@inkcre/core": "workspace:*" + }, + "devDependencies": { + "@inkcre/ext-dev-utils": "workspace:*", + "@inkcre/ui-web": "2.7.0", + "@module-federation/vite": "^1.1.0", + "@vitejs/plugin-vue": "^6.0.1", + "@vue/tsconfig": "catalog:", + "sass-embedded": "^1.90.0", + "typescript": "catalog:", + "vite": "^7.0.6", + "vue-tsc": "catalog:", + "zod": "^4.6.7" + }, + "peerDependencies": { + "@vueuse/core": "^14.0.0", + "pinia": "^3.0.0", + "vue": "^3.5.25", + "vue-router": "^4.5.0", + "zod": "^4.1.0" + }, + "inkcre": { + "name": "inkcre/bilibili", + "nickname": "哔哩哔哩", + "module_federation": { + "host_sdk": "@inkcre/core", + "host_sdk_version": ">=0.4.5 <0.5.0" + } + }, + "intent": { + "skills": [ + "@inkcre/ui-web" + ] + } +} diff --git a/extensions/bilibili/src/BilibiliContent.vue b/extensions/bilibili/src/BilibiliContent.vue new file mode 100644 index 00000000..3b41e9c4 --- /dev/null +++ b/extensions/bilibili/src/BilibiliContent.vue @@ -0,0 +1,175 @@ + + + + + diff --git a/extensions/bilibili/src/BilibiliPreview.vue b/extensions/bilibili/src/BilibiliPreview.vue new file mode 100644 index 00000000..5922dd9c --- /dev/null +++ b/extensions/bilibili/src/BilibiliPreview.vue @@ -0,0 +1,50 @@ + + + + + diff --git a/extensions/bilibili/src/BilibiliSetup.vue b/extensions/bilibili/src/BilibiliSetup.vue new file mode 100644 index 00000000..79e311b7 --- /dev/null +++ b/extensions/bilibili/src/BilibiliSetup.vue @@ -0,0 +1,600 @@ + + + + + diff --git a/extensions/bilibili/src/DevApp.vue b/extensions/bilibili/src/DevApp.vue new file mode 100644 index 00000000..543a743a --- /dev/null +++ b/extensions/bilibili/src/DevApp.vue @@ -0,0 +1,8 @@ + + + diff --git a/extensions/bilibili/src/collect.ts b/extensions/bilibili/src/collect.ts new file mode 100644 index 00000000..9fcb1dd2 --- /dev/null +++ b/extensions/bilibili/src/collect.ts @@ -0,0 +1,63 @@ +import type { FavoritePage } from './normalize.ts' +import type { BilibiliSourceConfig, CollectConfig, User } from './schema.ts' +import { + advanceProgress, + canResume, + emptyCursor, + verifyCompletion, + type Progress, +} from './progress.ts' + +export interface FavoriteSessionPort { + open(folder: string): Promise + account(): Promise + read(folder: string, page: number): Promise +} + +export async function collectFavorites( + config: BilibiliSourceConfig, + initial: Progress, + options: CollectConfig, + signal: AbortSignal, + session: FavoriteSessionPort, + commit: (account: User, page: FavoritePage, progress: Progress) => Promise +): Promise { + let progress = initial + for (const folder of config.folder_ids) { + signal.throwIfAborted() + let cursor = progress.folders[folder] ?? emptyCursor() + if (cursor.complete) continue + await session.open(folder) + await session.account() + let cachedHead: FavoritePage | null = null + if (cursor.next_page > 1) { + const head = await session.read(folder, 1) + const boundary = + cursor.next_page === 2 ? head : await session.read(folder, cursor.next_page - 1) + if (!canResume(cursor, head, boundary)) { + cursor = emptyCursor() + progress = { ...progress, folders: { ...progress.folders, [folder]: cursor } } + cachedHead = head + } + } + while (!cursor.complete && progress.last_run.pages < options.max_pages) { + signal.throwIfAborted() + const account = await session.account() + const current = cachedHead ?? (await session.read(folder, cursor.next_page)) + cachedHead = null + signal.throwIfAborted() + if (!current.hasMore) { + const finalHead = await session.read(folder, 1) + signal.throwIfAborted() + verifyCompletion(cursor, current, cursor.next_page, finalHead) + } + const next = advanceProgress(progress, folder, current, cursor.next_page, options.max_pages) + await commit(account, current, next) + // Persisted progress follows the page's confirmed graph writes. + progress = next + cursor = progress.folders[folder]! + } + if (progress.last_run.pages >= options.max_pages) break + } + return progress +} diff --git a/extensions/bilibili/src/graph.ts b/extensions/bilibili/src/graph.ts new file mode 100644 index 00000000..24ea0160 --- /dev/null +++ b/extensions/bilibili/src/graph.ts @@ -0,0 +1,50 @@ +import { resolverTypes, type User } from './schema.ts' +import type { FavoritePage } from './normalize.ts' + +export interface GraphRecord { + key: string + resolver: string + content: Record +} +export interface GraphRelation { + from_key: string + to_key: string + content: string +} +export interface SourceGraphBatch { + records: GraphRecord[] + relations: GraphRelation[] +} + +function knownFields(value: object): Record { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) +} + +export function favoriteGraph(account: User, page: FavoritePage): SourceGraphBatch { + const records = new Map() + const relations = new Map() + function record(key: string, resolver: string, value: object) { + const existing = records.get(key) + records.set(key, { key, resolver, content: { ...existing?.content, ...knownFields(value) } }) + } + function relate(from_key: string, to_key: string, content: string) { + relations.set(JSON.stringify([from_key, to_key, content]), { from_key, to_key, content }) + } + record(`user:${account.mid}`, resolverTypes.user, account) + record(`user:${page.owner.mid}`, resolverTypes.user, page.owner) + record(`folder:${page.folder.id}`, resolverTypes.folder, page.folder) + relate(`user:${account.mid}`, `folder:${page.folder.id}`, 'owns') + for (const item of page.items) { + const video = `video:${item.video.aid}` + const membership = `membership:${page.folder.id}:${item.video.aid}` + record(video, resolverTypes.video, item.video) + record(membership, resolverTypes.membership, item.membership) + relate(`folder:${page.folder.id}`, membership, 'contains') + relate(membership, video, 'item') + if (item.uploader) { + record(`user:${item.uploader.mid}`, resolverTypes.user, item.uploader) + relate(video, `user:${item.uploader.mid}`, 'author') + } + } + return { records: [...records.values()], relations: [...relations.values()] } +} diff --git a/extensions/bilibili/src/index.ts b/extensions/bilibili/src/index.ts new file mode 100644 index 00000000..0dda5b3f --- /dev/null +++ b/extensions/bilibili/src/index.ts @@ -0,0 +1,31 @@ +import { z } from 'zod' +import { Resolver, SourceManager, SourceType, type ExtensionModule } from '@inkcre/core' +import { UserResolver, FolderResolver, VideoResolver, MembershipResolver } from './resolver.ts' +import { SourceConfigSchema, CollectConfigSchema, sourceType } from './schema.ts' +import { bilibiliSource } from './source.ts' +import BilibiliSetup from './BilibiliSetup.vue' + +const resolvers = [UserResolver, FolderResolver, VideoResolver, MembershipResolver] +const extension: ExtensionModule = { + setup: { component: BilibiliSetup }, + async initialize() { + for (const resolver of resolvers) Resolver.register(resolver.type, resolver) + SourceManager.register(sourceType, bilibiliSource) + await SourceType.dbApi + .upsert({ + id: sourceType, + description: '采集自己的 B 站收藏夹，包括所选私密收藏夹。', + config_schema: z.toJSONSchema(SourceConfigSchema), + collect_config_schema: z.toJSONSchema(CollectConfigSchema), + backfill_config_schema: null, + }) + .throwOnError() + }, + async activate() {}, + async deactivate() {}, + async dispose() { + SourceManager.unregister(sourceType, bilibiliSource) + for (const resolver of resolvers) Resolver.unregister(resolver.type, resolver) + }, +} +export default extension diff --git a/extensions/bilibili/src/main.ts b/extensions/bilibili/src/main.ts new file mode 100644 index 00000000..a81fb0bc --- /dev/null +++ b/extensions/bilibili/src/main.ts @@ -0,0 +1,8 @@ +import { bootstrapExtension } from '@inkcre/ext-dev-utils' +import '@inkcre/ui-web/styles' +import App from './DevApp.vue' +import extensionModule from './index.ts' + +void bootstrapExtension({ rootComponent: App, extensionModule }).catch(() => + console.error('无法启动 B 站插件开发页面。') +) diff --git a/extensions/bilibili/src/normalize.ts b/extensions/bilibili/src/normalize.ts new file mode 100644 index 00000000..e1951773 --- /dev/null +++ b/extensions/bilibili/src/normalize.ts @@ -0,0 +1,200 @@ +import { z } from 'zod' +import { + FolderSchema, + MembershipSchema, + UserSchema, + VideoSchema, + nativeId, + type Folder, + type Membership, + type User, + type Video, +} from './schema.ts' + +const object = z.record(z.string(), z.unknown()) +export interface FavoriteItem { + video: Video + uploader: User | null + membership: Membership +} +export interface FavoritePage { + folder: Folder + owner: User + items: FavoriteItem[] + hasMore: boolean + unsupported: number + unresolved: number + totalItems: number + ids: string[] +} + +export function responseData(response: unknown): Record { + const envelope = object.parse(response) + if (envelope.code !== 0) { + const message = typeof envelope.message === 'string' ? envelope.message : 'B 站未返回成功结果。' + throw new Error(`B 站请求失败（${String(envelope.code)}）：${message}`) + } + return object.parse(envelope.data) +} + +function text(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined +} +function seconds(value: unknown): number | undefined { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined +} +function instant(value: unknown): string | undefined { + const timestamp = seconds(value) + if (!timestamp) return undefined + const date = new Date(timestamp * 1000) + return Number.isNaN(date.getTime()) ? undefined : date.toISOString() +} +function webUrl(value: unknown): string | undefined { + if (typeof value !== 'string' || !value) return undefined + try { + const url = new URL(value.startsWith('//') ? `https:${value}` : value) + return ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password + ? url.href + : undefined + } catch { + return undefined + } +} + +export function parseUser(value: unknown): User { + const user = object.parse(value) + return UserSchema.parse({ + mid: nativeId(user.mid), + name: text(user.name ?? user.uname), + avatar_url: webUrl(user.face), + }) +} + +export function parseAccount(response: unknown): User { + const data = responseData(response) + if (data.isLogin !== true) throw new Error('请先在所选浏览器登录 B 站，再读取收藏夹。') + return parseUser(data) +} + +export function parseFolder( + value: unknown, + expectedOwner: string +): { folder: Folder; owner: User } { + const info = object.parse(value) + const owner = parseUser(info.upper ?? { mid: info.mid }) + if (owner.mid !== expectedOwner || (info.mid != null && nativeId(info.mid) !== expectedOwner)) + throw new Error('收藏夹不属于已确认的 B 站账号。') + return { + owner, + folder: FolderSchema.parse({ + // info.fid is the original folder number; info.id is the full media_id used by the site. + id: nativeId(info.id), + title: text(info.title), + description: text(info.intro), + is_private: typeof info.attr === 'number' ? (info.attr & 1) === 1 : undefined, + reported_count: seconds(info.media_count), + }), + } +} + +export function parseFolders(response: unknown, owner: string): Folder[] { + const data = responseData(response) + if (!Array.isArray(data.list)) throw new Error('B 站未返回可识别的自建收藏夹列表。') + return data.list.map((value) => { + const row = object.parse(value) + return parseFolder({ ...row, upper: row.upper ?? { mid: row.mid ?? owner } }, owner).folder + }) +} + +export function parseFavoritePage( + response: unknown, + expectedOwner: string, + expectedFolder: string, + page: number, + observedAt = new Date().toISOString(), + pageSize = 20 +): FavoritePage { + const data = responseData(response) + const { folder, owner } = parseFolder(data.info, expectedOwner) + if (folder.id !== expectedFolder) throw new Error('B 站返回了另一个收藏夹的数据。') + if (typeof data.has_more !== 'boolean') + throw new Error('B 站分页结束标记缺失，无法确认采集范围。') + const medias = data.medias === null && !data.has_more ? [] : data.medias + if (!Array.isArray(medias)) throw new Error('B 站未返回可识别的收藏条目。') + if (medias.length > pageSize) + throw new Error('B 站返回了超出分页范围的条目，需重新核实分页协议。') + if (medias.length === 0 && data.has_more) + throw new Error('收藏分页为空但仍声明有后续，已停止采集。') + const items: FavoriteItem[] = [] + let unsupported = 0 + let unresolved = 0 + for (let index = 0; index < medias.length; index++) { + const raw = object.safeParse(medias[index]) + if (!raw.success) { + unresolved++ + continue + } + const value = raw.data + if (value.type !== 2) { + unsupported++ + continue + } + try { + const aid = nativeId(value.id) + const unavailable = + value.title === '已失效视频' || (typeof value.attr === 'number' && (value.attr & 1) === 1) + const bvid = z + .string() + .regex(/^BV[0-9A-Za-z]{10}$/) + .safeParse(value.bvid ?? value.bv_id) + let uploader: User | null = null + if (value.upper != null) { + try { + uploader = parseUser(value.upper) + } catch { + /* A missing uploader must not discard an identified video. */ + } + } + items.push({ + video: VideoSchema.parse({ + aid, + bvid: bvid.success ? bvid.data : undefined, + title: unavailable ? undefined : text(value.title), + description: unavailable ? undefined : text(value.intro), + duration_seconds: unavailable ? undefined : seconds(value.duration), + published_at: unavailable ? undefined : instant(value.pubtime), + cover_url: unavailable ? undefined : webUrl(value.cover), + }), + uploader, + membership: MembershipSchema.parse({ + availability: unavailable ? 'unavailable' : 'available', + page_number: page, + position_in_page: index, + favored_at: instant(value.fav_time), + observed_at: observedAt, + }), + }) + } catch { + unresolved++ + } + } + const ids = medias.map((value, index) => { + const row = object.safeParse(value) + if (!row.success) return `unresolved:${index}` + try { + return `${String(row.data.type)}:${nativeId(row.data.id)}` + } catch { + return `unresolved:${index}` + } + }) + return { + folder, + owner, + items, + hasMore: data.has_more, + unsupported, + unresolved, + totalItems: medias.length, + ids, + } +} diff --git a/extensions/bilibili/src/progress.ts b/extensions/bilibili/src/progress.ts new file mode 100644 index 00000000..36626636 --- /dev/null +++ b/extensions/bilibili/src/progress.ts @@ -0,0 +1,163 @@ +import { z } from 'zod' +import { SourceConfigSchema, type BilibiliSourceConfig } from './schema.ts' +import type { FavoritePage } from './normalize.ts' + +const CursorSchema = z.strictObject({ + next_page: z.number().int().positive(), + head_ids: z.array(z.string()).max(20), + boundary_ids: z.array(z.string()).max(20), + reported_count: z.number().int().nonnegative().optional(), + complete: z.boolean(), + rows: z.number().int().nonnegative(), + items: z.number().int().nonnegative().default(0), + unavailable: z.number().int().nonnegative().default(0), + unsupported: z.number().int().nonnegative().default(0), + unresolved: z.number().int().nonnegative().default(0), +}) +export const ProgressSchema = z.strictObject({ + version: z.literal(1), + scope: SourceConfigSchema, + folders: z.record(z.string(), CursorSchema), + last_run: z.strictObject({ + started_at: z.iso.datetime(), + updated_at: z.iso.datetime(), + pages: z.number().int().nonnegative(), + items: z.number().int().nonnegative(), + unavailable: z.number().int().nonnegative(), + unsupported: z.number().int().nonnegative(), + unresolved: z.number().int().nonnegative(), + status: z.enum(['running', 'partial', 'complete']), + }), +}) +export type Cursor = z.infer +export type Progress = z.infer + +export function emptyCursor(): Cursor { + return { + next_page: 1, + head_ids: [], + boundary_ids: [], + complete: false, + rows: 0, + items: 0, + unavailable: 0, + unsupported: 0, + unresolved: 0, + } +} + +export function scopeKey(config: BilibiliSourceConfig): string { + return JSON.stringify({ ...config, folder_ids: [...config.folder_ids].sort() }) +} + +export function startProgress( + state: Record, + config: BilibiliSourceConfig +): Progress { + const previous = ProgressSchema.safeParse(state.bilibili) + const sameScope = previous.success && scopeKey(previous.data.scope) === scopeKey(config) + const resume = sameScope && config.folder_ids.some((id) => !previous.data.folders[id]?.complete) + const now = new Date().toISOString() + return { + version: 1, + scope: config, + folders: Object.fromEntries( + config.folder_ids.map((id) => [ + id, + resume ? (previous.data.folders[id] ?? emptyCursor()) : emptyCursor(), + ]) + ), + last_run: { + started_at: now, + updated_at: now, + pages: 0, + items: 0, + unavailable: 0, + unsupported: 0, + unresolved: 0, + status: 'running', + }, + } +} + +export function sameIds(left: string[], right: string[]): boolean { + return JSON.stringify(left) === JSON.stringify(right) +} + +export function canResume(cursor: Cursor, head: FavoritePage, boundary: FavoritePage): boolean { + return ( + sameIds(cursor.head_ids, head.ids) && + cursor.reported_count === head.folder.reported_count && + sameIds(cursor.boundary_ids, boundary.ids) && + boundary.hasMore + ) +} + +export function verifyCompletion( + cursor: Cursor, + page: FavoritePage, + pageNumber: number, + head: FavoritePage +): void { + const expectedHead = pageNumber === 1 ? page.ids : cursor.head_ids + const reportedCount = pageNumber === 1 ? page.folder.reported_count : cursor.reported_count + if ( + !sameIds(expectedHead, head.ids) || + reportedCount !== head.folder.reported_count || + reportedCount !== page.folder.reported_count || + (pageNumber === 1 && head.hasMore !== page.hasMore) + ) { + throw new Error( + '收藏夹在采集期间发生变化，无法确认本轮分页已遍历；已确认的批次保留，请再次执行以重新核对。' + ) + } +} + +export function advanceProgress( + progress: Progress, + folder: string, + page: FavoritePage, + pageNumber: number, + maxPages: number +): Progress { + const previous = progress.folders[folder] ?? emptyCursor() + if (pageNumber > 1 && page.hasMore && sameIds(previous.boundary_ids, page.ids)) + throw new Error('B 站分页没有前进，已停止以避免将重复页误记为完整采集。') + if (pageNumber === 1 && page.totalItems === 0 && (page.folder.reported_count ?? 0) > 0) + throw new Error('收藏夹声明有条目但未返回内容，无法确认是空收藏夹。') + const folders = { + ...progress.folders, + [folder]: { + next_page: page.hasMore ? pageNumber + 1 : pageNumber, + head_ids: pageNumber === 1 ? page.ids : previous.head_ids, + boundary_ids: page.ids, + reported_count: pageNumber === 1 ? page.folder.reported_count : previous.reported_count, + complete: !page.hasMore, + rows: previous.rows + page.totalItems, + items: previous.items + page.items.length, + unavailable: + previous.unavailable + + page.items.filter((item) => item.membership.availability === 'unavailable').length, + unsupported: previous.unsupported + page.unsupported, + unresolved: previous.unresolved + page.unresolved, + }, + } + const pages = progress.last_run.pages + 1 + const complete = progress.scope.folder_ids.every((id) => folders[id]?.complete) + return { + ...progress, + folders, + last_run: { + ...progress.last_run, + updated_at: new Date().toISOString(), + pages, + items: progress.last_run.items + page.items.length, + unavailable: + progress.last_run.unavailable + + page.items.filter((item) => item.membership.availability === 'unavailable').length, + unsupported: progress.last_run.unsupported + page.unsupported, + unresolved: progress.last_run.unresolved + page.unresolved, + status: complete ? 'complete' : pages >= maxPages ? 'partial' : 'running', + }, + } +} diff --git a/extensions/bilibili/src/repository.ts b/extensions/bilibili/src/repository.ts new file mode 100644 index 00000000..783af44f --- /dev/null +++ b/extensions/bilibili/src/repository.ts @@ -0,0 +1,232 @@ +import { Block, Relation, Source } from '@inkcre/core' +import { NativeIdSchema, resolverTypes } from './schema.ts' +import type { GraphRecord, SourceGraphBatch } from './graph.ts' +import type { Progress } from './progress.ts' + +type StoredBlock = Pick + +function objectContent(content: string): Record | null { + try { + const value: unknown = JSON.parse(content) + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : null + } catch { + return null + } +} + +function nativeKey(resolver: string, content: Record): string | null { + const kind = + resolver === resolverTypes.user + ? 'user' + : resolver === resolverTypes.folder + ? 'folder' + : resolver === resolverTypes.video + ? 'video' + : null + if (!kind) return null + const identity = NativeIdSchema.safeParse( + content[kind === 'user' ? 'mid' : kind === 'folder' ? 'id' : 'aid'] + ) + return identity.success ? `${kind}:${identity.data}` : null +} + +/** Bilibili reconciles its own native objects over the ordinary shared graph. */ +export class BilibiliRepository { + private loaded = false + private readonly records = new Map() + private readonly blocks = new Map() + private readonly memberships = new Map>() + + private async load() { + if (this.loaded) return + let offset = 0 + while (true) { + const result = await Block.dbApi + .from() + .select('*', { count: 'exact' }) + .in('resolver', Object.values(resolverTypes)) + .is('storage', null) + .order('id', { ascending: true }) + .range(offset, offset + 999) + .throwOnError() + const rows = result.data ?? [] + for (const row of rows) { + this.blocks.set(row.id, row) + const content = objectContent(row.content) + const key = content && nativeKey(row.resolver, content) + if (key && !this.records.has(key)) this.records.set(key, row) + } + offset += rows.length + if (!rows.length || (result.count !== null ? offset >= result.count : rows.length < 1000)) + break + } + this.loaded = true + } + + private async saveRecord(record: GraphRecord): Promise { + const key = nativeKey(record.resolver, record.content) + if (!key) throw new Error('B 站对象缺少可确认的原生身份。') + const previous = this.records.get(key) + const saved = await this.saveContent(previous, record.resolver, record.content) + this.records.set(key, saved) + this.blocks.set(saved.id, saved) + return saved + } + + private async saveContent( + previous: StoredBlock | undefined, + resolver: string, + patch: Record + ) { + const original = previous ? objectContent(previous.content) : null + const value = { ...original, ...patch } + if (previous && JSON.stringify(original) === JSON.stringify(value)) return previous + const content = JSON.stringify(value) + // A lost create response ends this attempt. A later execution reads ordinary Blocks again. + const query = previous + ? Block.dbApi.update({ content }).eq('id', previous.id) + : Block.dbApi.insert({ resolver, storage: null, content }) + const result = await query.select().single().throwOnError() + return result.data + } + + private async folderMemberships(folder: number): Promise> { + const cached = this.memberships.get(folder) + if (cached) return cached + const found = new Map() + let offset = 0 + while (true) { + const result = await Relation.dbApi + .from() + .select('*', { count: 'exact' }) + .eq('from_', folder) + .eq('content', 'contains') + .order('id', { ascending: true }) + .range(offset, offset + 999) + .throwOnError() + const rows = result.data ?? [] + const ids = rows.flatMap((row) => (row.to_ === null ? [] : [row.to_])) + for (let index = 0; index < ids.length; index += 200) { + const items = await Relation.dbApi + .from() + .select() + .in('from_', ids.slice(index, index + 200)) + .eq('content', 'item') + .throwOnError() + for (const memberId of ids.slice(index, index + 200)) { + const membership = this.blocks.get(memberId) + const links = (items.data ?? []).filter((row) => row.from_ === memberId) + if (membership?.resolver !== resolverTypes.membership || links.length !== 1) continue + const video = links[0]!.to_ === null ? undefined : this.blocks.get(links[0]!.to_) + const content = video && objectContent(video.content) + const key = video && content && nativeKey(video.resolver, content) + if (!video || !key || this.records.get(key)?.id !== video.id) continue + if (!found.has(video.id)) found.set(video.id, membership) + } + } + offset += rows.length + if (!rows.length || (result.count !== null ? offset >= result.count : rows.length < 1000)) + break + } + this.memberships.set(folder, found) + return found + } + + private async link(from: number, to: number, content: string) { + const existing = await Relation.dbApi + .from() + .select('id') + .eq('from_', from) + .eq('to_', to) + .eq('content', content) + .limit(1) + .throwOnError() + if (existing.data?.length) return + await Relation.dbApi.insert({ from_: from, to_: to, content }).throwOnError() + } + + private async anchor(source: Source): Promise { + let anchor: StoredBlock | undefined + if (source.block !== null) { + const result = await Block.dbApi + .from() + .select() + .eq('id', source.block) + .single() + .throwOnError() + anchor = result.data + } else { + const result = await Block.dbApi + .from() + .select() + .eq('resolver', 'core.source.v1') + .order('id', { ascending: true }) + .throwOnError() + anchor = result.data?.find((row) => objectContent(row.content)?.id === source.id) + } + if ( + anchor && + (anchor.resolver !== 'core.source.v1' || objectContent(anchor.content)?.id !== source.id) + ) + throw new Error('采集源的来源锚点不匹配，已停止写入。') + return this.saveContent(anchor, 'core.source.v1', { + id: source.id, + type: source.type, + nickname: source.nickname, + }) + } + + async reconcilePage(source: Source, graph: SourceGraphBatch, progress: Progress): Promise { + await this.load() + const ids = new Map() + for (const record of graph.records) { + if (record.resolver === resolverTypes.membership) continue + ids.set(record.key, (await this.saveRecord(record)).id) + } + for (const record of graph.records) { + if (record.resolver !== resolverTypes.membership) continue + const folderKey = graph.relations.find( + (edge) => edge.content === 'contains' && edge.to_key === record.key + )?.from_key + const videoKey = graph.relations.find( + (edge) => edge.content === 'item' && edge.from_key === record.key + )?.to_key + const folder = folderKey && ids.get(folderKey) + const video = videoKey && ids.get(videoKey) + if (!folder || !video) throw new Error('收藏条目缺少明确的收藏夹或视频关联。') + const memberships = await this.folderMemberships(folder) + const membership = await this.saveContent( + memberships.get(video), + record.resolver, + record.content + ) + await this.link(membership.id, video, 'item') + await this.link(folder, membership.id, 'contains') + memberships.set(video, membership) + this.blocks.set(membership.id, membership) + ids.set(record.key, membership.id) + } + for (const edge of graph.relations) { + if (edge.content === 'contains' || edge.content === 'item') continue + const from = ids.get(edge.from_key) + const to = ids.get(edge.to_key) + if (!from || !to) throw new Error('B 站资料关联端点缺失。') + await this.link(from, to, edge.content) + } + const root = ids.get(`user:${progress.scope.account_mid}`) + if (!root) throw new Error('本页缺少已确认的 B 站账号。') + const anchor = await this.anchor(source) + await this.link(anchor.id, root, 'collects') + const state = { ...source.state, bilibili: progress } + await Source.dbApi + .update({ block: anchor.id, state }) + .eq('id', source.id) + .select('id') + .single() + .throwOnError() + source.block = anchor.id + source.state = state + } +} diff --git a/extensions/bilibili/src/resolver.ts b/extensions/bilibili/src/resolver.ts new file mode 100644 index 00000000..049b43e3 --- /dev/null +++ b/extensions/bilibili/src/resolver.ts @@ -0,0 +1,149 @@ +import { markRaw } from 'vue' +import { Block, Relation, Resolver, type ProjectionOptions } from '@inkcre/core' +import { + parseRecord, + recordTitle, + resolverTypes, + type BilibiliRecord, + type Membership, +} from './schema.ts' +import BilibiliContent from './BilibiliContent.vue' +import BilibiliPreview from './BilibiliPreview.vue' + +export type RelatedRecord = BilibiliRecord & { blockId: number; membership?: Membership } +export interface RelatedGroup { + title: string + role: 'folders' | 'videos' | 'author' | 'item' + items: RelatedRecord[] + nextOffset: number | null +} +export interface SolvedBilibili { + value: BilibiliRecord + groups: RelatedGroup[] +} + +async function recordBlock(block: Block): Promise { + const content = await block.getHydratedContent() + if (typeof content !== 'string') throw new Error('B 站资料应为文本记录。') + const value = parseRecord(block.resolver, JSON.parse(content)) + return value ? { ...value, blockId: block.id } : null +} + +async function edges(ids: number[], role: string, incoming = false, offset = 0, limit = 51) { + if (!ids.length) return [] + const result = await Relation.dbApi + .from() + .select() + .in(incoming ? 'to_' : 'from_', ids) + .eq('content', role) + .order('id', { ascending: true }) + .range(offset, offset + limit - 1) + .throwOnError() + return (result.data ?? []).map((row) => Relation.parse(row)) +} + +async function records(ids: number[]) { + const result = await Promise.all((await Block.getMany(ids)).map(recordBlock)) + return result.filter((value): value is RelatedRecord => value !== null) +} + +export abstract class BilibiliResolver extends Resolver { + static readonly previewRenderer = markRaw(BilibiliPreview) + static readonly solvedContentRenderer = markRaw(BilibiliContent) + + async readGroup(role: RelatedGroup['role'], offset = 0): Promise { + const value = parseRecord(this.block.resolver, JSON.parse(await this.getRawContent())) + if (!value) throw new Error('无法识别 B 站资料类型。') + let initial: Relation[] = [] + let title = '' + let items: RelatedRecord[] = [] + if (value.kind === 'user' && role === 'folders') { + title = '已保存的收藏夹' + initial = await edges([this.block.id], 'owns', false, offset) + items = (await records(initial.slice(0, 50).map((edge) => edge.to_))).filter( + (item) => item.kind === 'folder' + ) + } else if (value.kind === 'folder' && role === 'videos') { + title = '已保存的收藏条目' + initial = await edges([this.block.id], 'contains', false, offset) + const memberships = (await records(initial.slice(0, 50).map((edge) => edge.to_))).filter( + (item) => item.kind === 'membership' + ) + const links = await edges( + memberships.map((item) => item.blockId), + 'item', + false, + 0, + 100 + ) + const videos = await records(links.map((edge) => edge.to_)) + for (const membership of memberships) { + const link = links.find((edge) => edge.from_ === membership.blockId) + const video = videos.find((item) => item.blockId === link?.to_) + if (video?.kind === 'video' && membership.kind === 'membership') + items.push({ ...video, membership: membership.record }) + } + } else if (value.kind === 'video' && role === 'author') { + title = 'UP 主' + initial = await edges([this.block.id], 'author', false, offset) + items = (await records(initial.slice(0, 50).map((edge) => edge.to_))).filter( + (item) => item.kind === 'user' + ) + } else if (value.kind === 'video' && role === 'folders') { + title = '所在收藏夹' + initial = await edges([this.block.id], 'item', true, offset) + const membershipIds = initial.slice(0, 50).map((edge) => edge.from_) + const memberships = await records(membershipIds) + const links = await edges(membershipIds, 'contains', true, 0, 100) + const folders = await records(links.map((edge) => edge.from_)) + for (const membership of memberships) { + const link = links.find((edge) => edge.to_ === membership.blockId) + const folder = folders.find((item) => item.blockId === link?.from_) + if (folder?.kind === 'folder' && membership.kind === 'membership') + items.push({ ...folder, membership: membership.record }) + } + } else if (value.kind === 'membership' && role === 'item') { + title = '视频' + initial = await edges([this.block.id], 'item', false, offset) + items = (await records(initial.slice(0, 50).map((edge) => edge.to_))).filter( + (item) => item.kind === 'video' + ) + } else throw new Error('此资料没有所选关系。') + return { title, role, items, nextOffset: initial.length > 50 ? offset + 50 : null } + } + + protected async _getSolvedContent(options: ProjectionOptions): Promise { + const value = parseRecord(this.block.resolver, JSON.parse(await this.getRawContent(options))) + if (!value) throw new Error('无法识别 B 站资料类型。') + const roles: RelatedGroup['role'][] = + value.kind === 'user' + ? ['folders'] + : value.kind === 'folder' + ? ['videos'] + : value.kind === 'video' + ? ['author', 'folders'] + : ['item'] + return { value, groups: await Promise.all(roles.map((role) => this.readGroup(role))) } + } + + async getText(options: ProjectionOptions = {}): Promise { + const value = parseRecord(this.block.resolver, JSON.parse(await this.getRawContent(options))) + if (!value) throw new Error('无法识别 B 站资料类型。') + const description = + value.kind === 'video' || value.kind === 'folder' ? value.record.description : undefined + return [recordTitle(value), description].filter(Boolean).join('\n') + } +} + +export class UserResolver extends BilibiliResolver { + static readonly type = resolverTypes.user +} +export class FolderResolver extends BilibiliResolver { + static readonly type = resolverTypes.folder +} +export class VideoResolver extends BilibiliResolver { + static readonly type = resolverTypes.video +} +export class MembershipResolver extends BilibiliResolver { + static readonly type = resolverTypes.membership +} diff --git a/extensions/bilibili/src/schema.ts b/extensions/bilibili/src/schema.ts new file mode 100644 index 00000000..e3591ad7 --- /dev/null +++ b/extensions/bilibili/src/schema.ts @@ -0,0 +1,121 @@ +import { z } from 'zod' + +export const extensionName = 'inkcre/bilibili' +export const sourceType = 'extensions.bilibili.favorites.v1' +export const resolverTypes = { + user: 'extensions.bilibili.user.v1', + folder: 'extensions.bilibili.folder.v1', + video: 'extensions.bilibili.video.v1', + membership: 'extensions.bilibili.membership.v1', +} as const + +export const NativeIdSchema = z.string().regex(/^[1-9]\d*$/) +const instant = z.iso.datetime() +export const UserSchema = z.strictObject({ + mid: NativeIdSchema, + name: z.string().optional(), + avatar_url: z.url().optional(), +}) +export const FolderSchema = z.strictObject({ + id: NativeIdSchema, + title: z.string().optional(), + description: z.string().optional(), + is_private: z.boolean().optional(), + reported_count: z.number().int().nonnegative().optional(), +}) +export const VideoSchema = z.strictObject({ + aid: NativeIdSchema, + bvid: z + .string() + .regex(/^BV[0-9A-Za-z]{10}$/) + .optional(), + title: z.string().optional(), + description: z.string().optional(), + duration_seconds: z.number().int().nonnegative().optional(), + published_at: instant.optional(), + cover_url: z.url().optional(), +}) +// Availability and bookmark time describe this account's list item, not the global video. +export const MembershipSchema = z.strictObject({ + availability: z.enum(['available', 'unavailable', 'unknown']), + page_number: z.number().int().positive().optional(), + position_in_page: z.number().int().nonnegative().optional(), + favored_at: instant.optional(), + observed_at: instant, +}) +export const SourceConfigSchema = z.strictObject({ + provider: z.uuid(), + account_mid: NativeIdSchema, + folder_ids: z + .array(NativeIdSchema) + .min(1) + .max(128) + .refine((ids) => new Set(ids).size === ids.length), +}) +export const CollectConfigSchema = z.strictObject({ + max_pages: z.number().int().min(1).max(100).default(10), +}) +export type User = z.infer +export type Folder = z.infer +export type Video = z.infer +export type Membership = z.infer +export type BilibiliSourceConfig = z.infer +export type CollectConfig = z.infer + +export function nativeId(value: unknown): string { + if (typeof value === 'number') { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error('来源身份不是安全整数。') + return String(value) + } + return NativeIdSchema.parse(value) +} + +export function videoUrl(video: Video): string { + return `https://www.bilibili.com/video/${video.bvid ?? `av${video.aid}`}` +} + +export function safeWebUrl(value: string | undefined): string | null { + if (!value) return null + try { + const url = new URL(value) + return ['https:', 'http:'].includes(url.protocol) && !url.username && !url.password + ? url.href + : null + } catch { + return null + } +} + +export type BilibiliRecord = + | { kind: 'user'; record: User } + | { kind: 'folder'; record: Folder } + | { kind: 'video'; record: Video } + | { kind: 'membership'; record: Membership } + +export function recordTitle(value: BilibiliRecord): string { + switch (value.kind) { + case 'user': + return value.record.name || `UID ${value.record.mid}` + case 'folder': + return value.record.title || `收藏夹 ${value.record.id}` + case 'video': + return value.record.title || value.record.bvid || `av${value.record.aid}` + case 'membership': + return value.record.availability === 'unavailable' ? '已失效的收藏条目' : '收藏条目' + } +} + +export function parseRecord(resolver: string, content: unknown): BilibiliRecord | null { + switch (resolver) { + case resolverTypes.user: + return { kind: 'user', record: UserSchema.parse(content) } + case resolverTypes.folder: + return { kind: 'folder', record: FolderSchema.parse(content) } + case resolverTypes.video: + return { kind: 'video', record: VideoSchema.parse(content) } + case resolverTypes.membership: + return { kind: 'membership', record: MembershipSchema.parse(content) } + default: + return null + } +} diff --git a/extensions/bilibili/src/site.ts b/extensions/bilibili/src/site.ts new file mode 100644 index 00000000..38f3b911 --- /dev/null +++ b/extensions/bilibili/src/site.ts @@ -0,0 +1,77 @@ +import { BrowserAccess, type BrowserPage } from '@inkcre/core' +import { parseAccount, parseFavoritePage, parseFolders } from './normalize.ts' +import type { BilibiliSourceConfig } from './schema.ts' + +const apiOrigin = 'https://api.bilibili.com' + +async function readJson(page: BrowserPage, path: string, parameters: Record = {}) { + const url = new URL(path, apiOrigin) + url.search = new URLSearchParams(parameters).toString() + return page.evaluate(async (address) => { + // This runs in the selected logged-in tab; credentials never leave the browser profile. + const response = await fetch(address, { credentials: 'include' }) + if (!response.ok) throw new Error(`B 站请求失败（HTTP ${response.status}）。`) + const value: unknown = await response.json() + return value + }, url.href) +} + +export async function readAccount(page: BrowserPage) { + return parseAccount(await readJson(page, '/x/web-interface/nav')) +} + +export async function discoverFavorites(provider: string, signal: AbortSignal) { + return BrowserAccess.withPage({ provider, target: { kind: 'new' }, signal }, async (page) => { + await page.goto('https://www.bilibili.com/', { waitUntil: 'domcontentloaded' }) + signal.throwIfAborted() + const account = await readAccount(page) + const folders = parseFolders( + await readJson(page, '/x/v3/fav/folder/created/list-all', { + up_mid: account.mid, + }), + account.mid + ) + signal.throwIfAborted() + return { account, folders } + }) +} + +export class FavoriteSession { + constructor( + private readonly page: BrowserPage, + readonly config: BilibiliSourceConfig + ) {} + + async open(folder: string) { + await this.page.goto( + `https://space.bilibili.com/${this.config.account_mid}/favlist?fid=${folder}&ftype=create`, + { + waitUntil: 'domcontentloaded', + } + ) + } + + async account() { + const account = await readAccount(this.page) + if (account.mid !== this.config.account_mid) + throw new Error('所选浏览器的 B 站账号已改变，请重新确认采集源。') + return account + } + + async read(folder: string, pageNumber: number) { + return parseFavoritePage( + await readJson(this.page, '/x/v3/fav/resource/list', { + media_id: folder, + pn: String(pageNumber), + ps: '20', + order: 'mtime', + type: '0', + tid: '0', + platform: 'web', + }), + this.config.account_mid, + folder, + pageNumber + ) + } +} diff --git a/extensions/bilibili/src/source.ts b/extensions/bilibili/src/source.ts new file mode 100644 index 00000000..ca5a3e5b --- /dev/null +++ b/extensions/bilibili/src/source.ts @@ -0,0 +1,35 @@ +import { BrowserAccess, type SourceImplementation } from '@inkcre/core' +import { CollectConfigSchema, SourceConfigSchema } from './schema.ts' +import { favoriteGraph } from './graph.ts' +import { startProgress } from './progress.ts' +import { BilibiliRepository } from './repository.ts' +import { FavoriteSession } from './site.ts' +import { collectFavorites } from './collect.ts' + +export const bilibiliSource: SourceImplementation = { + collectConfig: CollectConfigSchema, + backfillConfig: null, + canCollect(source) { + const config = SourceConfigSchema.safeParse(source.config) + return config.success && BrowserAccess.hasBinding(config.data.provider) + }, + async collect(source, _job, options, signal) { + const config = SourceConfigSchema.parse(source.config) + const repository = new BilibiliRepository() + await BrowserAccess.withPage( + { provider: config.provider, target: { kind: 'new' }, signal }, + async (page) => { + await collectFavorites( + config, + startProgress(source.state, config), + options, + signal, + new FavoriteSession(page, config), + async (account, current, next) => { + await repository.reconcilePage(source, favoriteGraph(account, current), next) + } + ) + } + ) + }, +} diff --git a/extensions/bilibili/tests/favorites.test.ts b/extensions/bilibili/tests/favorites.test.ts new file mode 100644 index 00000000..fbd777dc --- /dev/null +++ b/extensions/bilibili/tests/favorites.test.ts @@ -0,0 +1,366 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + parseAccount, + parseFavoritePage, + parseFolders, + type FavoritePage, +} from '../src/normalize.ts' +import { favoriteGraph } from '../src/graph.ts' +import { SourceConfigSchema, nativeId } from '../src/schema.ts' +import { startProgress, type Progress } from '../src/progress.ts' +import { collectFavorites, type FavoriteSessionPort } from '../src/collect.ts' + +// Invented identities and text; only response field shapes come from observed browser responses. +const account = { mid: '1001', name: '测试账号' } +const info = { + id: 900123, + fid: 9001, + mid: 1001, + upper: account, + title: '技术收藏', + attr: 1, + media_count: 41, +} +const config = SourceConfigSchema.parse({ + provider: 'd089e2c4-e3ec-42a6-953b-42530a726ac2', + account_mid: account.mid, + folder_ids: ['900123'], +}) +const media = (id: number, extra: object = {}) => ({ + id, + type: 2, + bvid: 'BV1234567890', + title: `视频 ${id}`, + intro: '已知简介', + attr: 0, + upper: { mid: 2002, name: '测试 UP 主' }, + fav_time: 1700000000, + ...extra, +}) +function page(ids: number[], more: boolean, number = 1, extra: object = {}): FavoritePage { + return parseFavoritePage( + { code: 0, data: { info, medias: ids.map((id) => media(id)), has_more: more, ...extra } }, + account.mid, + '900123', + number, + '2026-11-06T00:00:00.000Z' + ) +} + +void test('full media_id is kept distinct from raw fid, visibility is a bit, and login is required', () => { + const folders = parseFolders( + { code: 0, data: { list: [info, { ...info, id: 900124, attr: 23 }] } }, + account.mid + ) + assert.equal(folders[0]!.id, '900123') + assert.equal(folders[0]!.is_private, true) + assert.equal(folders[1]!.is_private, true) + assert.throws(() => parseAccount({ code: 0, data: { isLogin: false, mid: 1001 } }), /登录/) + assert.throws( + () => parseFolders({ code: 0, data: { list: [{ ...info, mid: 777 }] } }, account.mid), + /不属于/ + ) + assert.throws(() => nativeId(Number.MAX_SAFE_INTEGER + 1), /安全整数/) +}) + +void test('short pages follow has_more, unavailable placeholders preserve unknown fields, and types are reported', () => { + const value = parseFavoritePage( + { + code: 0, + data: { + info, + has_more: true, + medias: [ + media(11), + media(12, { title: '已失效视频', attr: 9 }), + media(13, { type: 12 }), + { id: 'bad', type: 2 }, + ], + }, + }, + account.mid, + '900123', + 2, + '2026-11-06T00:00:00.000Z' + ) + assert.equal(value.hasMore, true) + assert.equal(value.items.length, 2) + assert.equal(value.unsupported, 1) + assert.equal(value.unresolved, 1) + assert.equal(value.items[1]!.video.title, undefined) + assert.equal(value.items[1]!.video.description, undefined) + assert.equal(value.items[1]!.membership.availability, 'unavailable') + assert.equal(value.items[1]!.membership.page_number, 2) + assert.equal(value.items[1]!.membership.position_in_page, 1) + const record = favoriteGraph(account, value).records.find((entry) => entry.key === 'video:12')! + assert.equal(Object.hasOwn(record.content, 'title'), false) + assert.throws(() => page([], true), /分页为空/) + assert.throws( + () => parseFavoritePage({ code: -101, message: '账号未登录' }, account.mid, '900123', 1), + /未登录/ + ) + assert.throws( + () => parseFavoritePage({ code: 0, data: { info, medias: [] } }, account.mid, '900123', 1), + /结束标记/ + ) +}) + +void test('a shared video has one identity and separate stable membership records in each folder', () => { + const sharedUploader = favoriteGraph(account, page([11, 12], false)) + assert.equal(sharedUploader.records.length, 7) + assert.equal( + new Set(sharedUploader.records.map((entry) => entry.key)).size, + sharedUploader.records.length + ) + const first = favoriteGraph(account, page([11], false)) + const secondPage = { ...page([11], false), folder: { ...page([11], false).folder, id: '900124' } } + const second = favoriteGraph(account, secondPage) + assert.equal( + first.records.find((entry) => entry.resolver.endsWith('.video.v1'))!.key, + second.records.find((entry) => entry.resolver.endsWith('.video.v1'))!.key + ) + assert.notEqual( + first.records.find((entry) => entry.resolver.endsWith('.membership.v1'))!.key, + second.records.find((entry) => entry.resolver.endsWith('.membership.v1'))!.key + ) + assert.deepEqual( + first.relations.map((entry) => entry.content), + ['owns', 'contains', 'item', 'author'] + ) +}) + +void test('optional malformed uploader and time do not discard an identified video', () => { + const value = parseFavoritePage( + { + code: 0, + data: { + info, + has_more: false, + medias: [media(11, { upper: { mid: 0 }, pubtime: Number.MAX_SAFE_INTEGER })], + }, + }, + account.mid, + '900123', + 1 + ) + assert.equal(value.items[0]!.video.aid, '11') + assert.equal(value.items[0]!.video.published_at, undefined) + assert.equal(value.items[0]!.uploader, null) + assert.equal(value.unresolved, 0) +}) + +function session(pages: FavoritePage[], requested: number[]): FavoriteSessionPort { + return { + open: async () => {}, + account: async () => account, + read: async (_folder, number) => { + requested.push(number) + return pages[number - 1]! + }, + } +} + +void test('budgeted runs continue beyond initial pages only after checking head and committed boundary', async () => { + const pages = [page([11, 12], true), page([13], true, 2), page([14], false, 3)] + const requested: number[] = [] + let persisted: Record = {} + const commit = async (_account: unknown, _page: unknown, progress: Progress) => { + persisted = { bilibili: progress } + } + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 2 }, + new AbortController().signal, + session(pages, requested), + commit + ) + assert.equal((persisted.bilibili as Progress).last_run.status, 'partial') + assert.equal((persisted.bilibili as Progress).folders['900123']!.next_page, 3) + requested.length = 0 + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 2 }, + new AbortController().signal, + session(pages, requested), + commit + ) + assert.deepEqual(requested, [1, 2, 3, 1]) + assert.equal((persisted.bilibili as Progress).last_run.status, 'complete') +}) + +void test('a changed head restarts scanning and failed persistence never advances acknowledged progress', async () => { + const pages = [page([11], true), page([12], false, 2)] + let persisted: Record = {} + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 1 }, + new AbortController().signal, + session(pages, []), + async (_account, _page, progress) => { + persisted = { bilibili: progress } + } + ) + const before = structuredClone(persisted) + const requested: number[] = [] + const changed = [page([99], true), page([11], false, 2)] + await assert.rejects( + collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 1 }, + new AbortController().signal, + session(changed, requested), + async () => { + throw new Error('page write failed') + } + ), + /page write failed/ + ) + assert.deepEqual(requested, [1]) + assert.deepEqual(persisted, before) + const imported: string[] = [] + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 1 }, + new AbortController().signal, + session(changed, []), + async (_account, current, progress) => { + imported.push(...current.ids) + persisted = { bilibili: progress } + } + ) + assert.deepEqual(imported, ['2:99']) +}) + +void test('cancellation and repeated pagination fail without a false completed checkpoint', async () => { + const controller = new AbortController() + controller.abort() + await assert.rejects( + collectFavorites( + config, + startProgress({}, config), + { max_pages: 2 }, + controller.signal, + session([], []), + async () => assert.fail('must not write') + ) + ) + let persisted: Record = {} + await assert.rejects( + collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 2 }, + new AbortController().signal, + session([page([11], true), page([11], true, 2)], []), + async (_account, _page, progress) => { + persisted = { bilibili: progress } + } + ), + /分页没有前进/ + ) + assert.equal((persisted.bilibili as Progress).folders['900123']!.next_page, 2) + assert.notEqual((persisted.bilibili as Progress).last_run.status, 'complete') +}) + +void test('a changed committed boundary restarts even when the first page has not changed', async () => { + const pages = [page([11], true), page([12], true, 2), page([13], false, 3)] + let persisted: Record = {} + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 2 }, + new AbortController().signal, + session(pages, []), + async (_account, _page, progress) => { + persisted = { bilibili: progress } + } + ) + const requested: number[] = [] + const changed = [pages[0]!, page([99], true, 2), pages[2]!] + const imported: string[] = [] + await collectFavorites( + config, + startProgress(persisted, config), + { max_pages: 1 }, + new AbortController().signal, + session(changed, requested), + async (_account, current, progress) => { + imported.push(...current.ids) + persisted = { bilibili: progress } + } + ) + assert.deepEqual(requested, [1, 2]) + assert.deepEqual(imported, ['2:11']) + assert.equal((persisted.bilibili as Progress).folders['900123']!.next_page, 2) +}) + +void test('an abort during page reading prevents dispatching an uncommitted batch', async () => { + const controller = new AbortController() + const target = session([page([11], false)], []) + target.read = async () => { + controller.abort() + return page([11], false) + } + await assert.rejects( + collectFavorites( + config, + startProgress({}, config), + { max_pages: 1 }, + controller.signal, + target, + async () => assert.fail('must not write an aborted batch') + ) + ) +}) + +void test('a head reorder during the same run prevents a false completed scan', async () => { + let persisted: Record = {} + let headReads = 0 + const target: FavoriteSessionPort = { + open: async () => {}, + account: async () => account, + read: async (_folder, number) => + number === 1 ? page([++headReads === 1 ? 11 : 99], true) : page([12], false, 2), + } + await assert.rejects( + collectFavorites( + config, + startProgress({}, config), + { max_pages: 2 }, + new AbortController().signal, + target, + async (_account, _page, next) => { + persisted = { bilibili: next } + } + ), + /采集期间发生变化/ + ) + const cursor = (persisted.bilibili as Progress).folders['900123']! + assert.equal(cursor.next_page, 2) + assert.equal(cursor.complete, false) + assert.equal(cursor.rows, 1) +}) + +void test('reported count differences from hidden items do not turn a fully traversed scan into an endless restart', async () => { + const target = session([page([11], true), page([12], false, 2)], []) + let result: Progress | undefined + await collectFavorites( + config, + startProgress({}, config), + { max_pages: 2 }, + new AbortController().signal, + target, + async (_account, _page, next) => { + result = next + } + ) + assert.equal(result!.last_run.status, 'complete') + assert.equal(result!.folders['900123']!.reported_count, 41) + assert.equal(result!.folders['900123']!.rows, 2) +}) diff --git a/extensions/bilibili/tests/repository.test.ts b/extensions/bilibili/tests/repository.test.ts new file mode 100644 index 00000000..f171de69 --- /dev/null +++ b/extensions/bilibili/tests/repository.test.ts @@ -0,0 +1,375 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { Block, Relation, Source } from '@inkcre/core' +import { BilibiliRepository } from '../src/repository.ts' +import { favoriteGraph } from '../src/graph.ts' +import { advanceProgress, startProgress } from '../src/progress.ts' +import { resolverTypes, SourceConfigSchema, sourceType } from '../src/schema.ts' +import type { FavoritePage } from '../src/normalize.ts' + +type Table = 'blocks' | 'relations' | 'sources' +type Row = { id: number; [key: string]: unknown } +type Values = Record +class Query { + table: Table + database: Database + action: 'read' | 'insert' | 'update' = 'read' + values: Values = {} + filters: ((row: Row) => boolean)[] = [] + start = 0 + end = Infinity + one = false + counted = false + constructor(database: Database, table: Table) { + this.database = database + this.table = table + } + select(_fields = '*', options?: { count: string }) { + this.counted = options?.count === 'exact' + return this + } + eq(key: string, value: unknown) { + this.filters.push((row) => row[key] === value) + return this + } + is(key: string, value: unknown) { + return this.eq(key, value) + } + in(key: string, values: unknown[]) { + this.filters.push((row) => values.includes(row[key])) + return this + } + order() { + return this + } + range(start: number, end: number) { + this.start = start + this.end = end + return this + } + limit(limit: number) { + this.end = this.start + limit - 1 + return this + } + single() { + this.one = true + return this + } + async throwOnError() { + return this.database.execute(this) + } +} +class Database { + rows: Record = { blocks: [], relations: [], sources: [] } + events: { table: Table; action: string; values: Values }[] = [] + failBefore: ((query: Query) => boolean) | null = null + failAfter: ((query: Query) => boolean) | null = null + pageLimit = 500 + client(table: Table) { + return { + from: () => new Query(this, table), + insert: (values: Values) => { + const query = new Query(this, table) + query.action = 'insert' + query.values = values + return query + }, + update: (values: Values) => { + const query = new Query(this, table) + query.action = 'update' + query.values = values + return query + }, + } + } + execute(query: Query) { + this.events.push({ + table: query.table, + action: query.action, + values: structuredClone(query.values), + }) + if (this.failBefore?.(query)) throw new Error('write failed') + const table = this.rows[query.table] + let result = table + .filter((row) => query.filters.every((filter) => filter(row))) + .sort((a, b) => a.id - b.id) + const count = result.length + if (query.action === 'insert') { + const row = { + id: Math.max(0, ...table.map((value) => value.id)) + 1, + ...structuredClone(query.values), + } + table.push(row) + result = [row] + } else if (query.action === 'update') { + for (const row of result) Object.assign(row, structuredClone(query.values)) + } + if (this.failAfter?.(query)) throw new Error('create outcome unknown') + if (query.action === 'read') + result = result.slice(query.start, Math.min(query.end + 1, query.start + this.pageLimit)) + if (query.one && result.length !== 1) throw new Error('Expected one result') + return { + data: structuredClone(query.one ? result[0] : result), + count: query.counted ? count : null, + } + } +} +const account = { mid: '1001', name: '当前名称' } +const config = SourceConfigSchema.parse({ + provider: 'd089e2c4-e3ec-42a6-953b-42530a726ac2', + account_mid: account.mid, + folder_ids: ['900123'], +}) +function page(): FavoritePage { + return { + folder: { id: '900123', title: '技术收藏', reported_count: 1 }, + owner: account, + items: [ + { + video: { aid: '11' }, + uploader: { mid: '2002', name: 'UP 主' }, + membership: { availability: 'unavailable', observed_at: '2026-11-07T00:00:00.000Z' }, + }, + ], + hasMore: false, + unsupported: 0, + unresolved: 0, + totalItems: 1, + ids: ['2:11'], + } +} +function source() { + return { + id: 1, + type: sourceType, + nickname: 'B 站', + config, + state: { kept: true }, + block: null, + } as unknown as Source +} +function next(source: Source) { + return advanceProgress(startProgress(source.state, config), '900123', page(), 1, 1) +} +function block(id: number, resolver: string, content: Values): Row { + return { id, resolver, storage: null, content: JSON.stringify(content, null, 2) } +} +function attach(context: { after: (action: () => void) => void }, database: Database) { + const originals = [Block.dbApi, Relation.dbApi, Source.dbApi] + Block.dbApi = database.client('blocks') as unknown as typeof Block.dbApi + Relation.dbApi = database.client('relations') as unknown as typeof Relation.dbApi + Source.dbApi = database.client('sources') as unknown as typeof Source.dbApi + context.after(() => { + ;[Block.dbApi, Relation.dbApi, Source.dbApi] = originals as typeof originals + }) +} +function savedGraph(database: Database, value: Source) { + const folder = database.rows.blocks.find((row) => row.resolver === resolverTypes.folder)! + const video = database.rows.blocks.find( + (row) => row.resolver === resolverTypes.video && JSON.parse(String(row.content)).aid === '11' + )! + const memberLink = database.rows.relations.find( + (row) => row.from_ === folder.id && row.content === 'contains' + )! + const item = database.rows.relations.find( + (row) => row.from_ === memberLink.to_ && row.content === 'item' + )! + assert.equal(item.to_, video.id) + const anchor = database.rows.blocks.find((row) => row.id === value.block)! + assert.equal(anchor.resolver, 'core.source.v1') + assert.equal(JSON.parse(String(anchor.content)).id, value.id) +} + +void test('ordinary pretty-printed JSON from another Peer is reused and repeat collection preserves IDs and user relations', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + value.block = 15 + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.rows.blocks.push( + block(10, resolverTypes.user, { + mid: '1001', + name: '原名称', + avatar_url: 'https://example.com/avatar.png', + }), + block(11, resolverTypes.folder, { id: '900123', title: '旧夹名' }), + block(12, resolverTypes.video, { aid: '11', title: '已知标题', description: '已知简介' }), + block(13, resolverTypes.user, { mid: '2002', name: 'UP 主' }), + block(14, resolverTypes.membership, { + availability: 'available', + favored_at: '2025-01-01T00:00:00.000Z', + observed_at: '2025-01-01T00:00:00.000Z', + }), + block(15, 'core.source.v1', { id: 1, type: sourceType, nickname: 'B 站' }), + { id: 16, resolver: 'core.text.v1', storage: null, content: '用户笔记' }, + block(17, resolverTypes.user, { mid: '11001', name: '提到 1001 的另一个人' }), + block(18, resolverTypes.video, { aid: '111', title: '提到 11 的另一个视频' }) + ) + database.rows.relations.push( + { id: 1, from_: 10, to_: 11, content: 'owns' }, + { id: 2, from_: 11, to_: 14, content: 'contains' }, + { id: 3, from_: 14, to_: 12, content: 'item' }, + { id: 4, from_: 12, to_: 13, content: 'author' }, + { id: 5, from_: 15, to_: 10, content: 'collects' }, + { id: 6, from_: 12, to_: 16, content: 'user:note' } + ) + const ids = database.rows.blocks.map((row) => row.id) + const userEdge = structuredClone(database.rows.relations[5]) + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + assert.deepEqual( + database.rows.blocks.map((row) => row.id), + ids + ) + assert.deepEqual( + database.rows.relations.find((row) => row.id === 6), + userEdge + ) + assert.equal(database.rows.relations.length, 6) + assert.equal( + JSON.parse(String(database.rows.blocks.find((row) => row.id === 10)!.content)).avatar_url, + 'https://example.com/avatar.png' + ) + assert.equal( + JSON.parse(String(database.rows.blocks.find((row) => row.id === 12)!.content)).title, + '已知标题' + ) + assert.equal( + JSON.parse(String(database.rows.blocks.find((row) => row.id === 14)!.content)).favored_at, + '2025-01-01T00:00:00.000Z' + ) + assert.equal(value.block, 15) + savedGraph(database, value) +}) + +void test('an existing membership for another video is never reused', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.rows.blocks.push( + block(10, resolverTypes.folder, { id: '900123' }), + block(11, resolverTypes.video, { aid: '99' }), + block(12, resolverTypes.membership, { + availability: 'available', + observed_at: '2025-01-01T00:00:00.000Z', + }) + ) + database.rows.relations.push( + { id: 1, from_: 10, to_: 12, content: 'contains' }, + { id: 2, from_: 12, to_: 11, content: 'item' } + ) + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + const currentVideo = database.rows.blocks.find( + (row) => row.resolver === resolverTypes.video && JSON.parse(String(row.content)).aid === '11' + )! + const currentItem = database.rows.relations.find( + (row) => row.content === 'item' && row.to_ === currentVideo.id + )! + assert.notEqual(currentItem.from_, 12) + assert.equal(database.rows.relations.find((row) => row.id === 2)!.to_, 11) +}) + +void test('partial graph failure leaves progress unchanged and the next execution repairs the ordinary graph', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.failBefore = (query) => + query.table === 'relations' && query.action === 'insert' && query.values.content === 'contains' + await assert.rejects( + new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)), + /write failed/ + ) + assert.deepEqual(value.state, { kept: true }) + assert.deepEqual(database.rows.sources[0]!.state, { kept: true }) + const originalVideo = database.rows.blocks.find((row) => row.resolver === resolverTypes.video)!.id + const incompleteMember = database.rows.blocks.find( + (row) => row.resolver === resolverTypes.membership + )!.id + assert.ok( + database.rows.relations.some((row) => row.from_ === incompleteMember && row.content === 'item') + ) + database.failBefore = null + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + assert.equal( + database.rows.blocks.find((row) => row.resolver === resolverTypes.video)!.id, + originalVideo + ) + assert.ok(database.rows.blocks.some((row) => row.id === incompleteMember)) + assert.ok(Object.hasOwn(value.state, 'bilibili')) + savedGraph(database, value) +}) + +void test('a lost create response is not replayed and a later execution reuses the visible native object', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.failAfter = (query) => + query.table === 'blocks' && + query.action === 'insert' && + query.values.resolver === resolverTypes.video + await assert.rejects( + new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)), + /create outcome unknown/ + ) + const created = database.rows.blocks.find((row) => row.resolver === resolverTypes.video)!.id + assert.deepEqual(value.state, { kept: true }) + const creates = () => + database.events.filter( + (event) => event.action === 'insert' && event.values.resolver === resolverTypes.video + ).length + assert.equal(creates(), 1) + database.failAfter = null + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + assert.equal(creates(), 1) + assert.equal( + database.rows.blocks.find((row) => row.resolver === resolverTypes.video)!.id, + created + ) + savedGraph(database, value) +}) + +void test('failure of the final Source update reuses the already saved graph and unbound Source anchor on the next execution', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + database.failBefore = (query) => query.table === 'sources' && query.action === 'update' + await assert.rejects( + new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)), + /write failed/ + ) + assert.deepEqual(value.state, { kept: true }) + assert.equal(value.block, null) + const ids = database.rows.blocks.map((row) => row.id) + const relations = structuredClone(database.rows.relations) + database.failBefore = null + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + assert.deepEqual( + database.rows.blocks.map((row) => row.id), + ids + ) + assert.deepEqual(database.rows.relations, relations) + savedGraph(database, value) +}) + +void test('normal paginated graph reads find existing objects beyond the service row cap', async (context) => { + const database = new Database() + attach(context, database) + const value = source() + database.rows.sources.push(structuredClone(value) as unknown as Row) + for (let id = 1; id <= 1100; id++) + database.rows.blocks.push(block(id, resolverTypes.user, { mid: String(1000000 + id) })) + database.rows.blocks.push(block(1101, resolverTypes.user, account)) + await new BilibiliRepository().reconcilePage(value, favoriteGraph(account, page()), next(value)) + const users = database.rows.blocks.filter( + (row) => + row.resolver === resolverTypes.user && JSON.parse(String(row.content)).mid === account.mid + ) + assert.deepEqual( + users.map((row) => row.id), + [1101] + ) +}) diff --git a/extensions/bilibili/tsconfig.json b/extensions/bilibili/tsconfig.json new file mode 100644 index 00000000..e7b23556 --- /dev/null +++ b/extensions/bilibili/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": ["@vue/tsconfig/tsconfig.dom.json", "../../tsconfig.browser.json"], + "compilerOptions": { + "types": ["vite/client"], + "outDir": "./dist/types", + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo", + "jsx": "preserve", + "jsxImportSource": "vue", + "skipLibCheck": true, + "paths": { + "@/*": ["../../apps/web/src/*"], + "@inkcre/core": ["../../packages/core/src/index.ts"], + "@inkcre/core/*": ["../../packages/core/src/*"] + }, + "allowImportingTsExtensions": true, + "strict": true + }, + "include": ["src/**/*", "src/**/*.vue"] +} diff --git a/extensions/bilibili/vite.config.ts b/extensions/bilibili/vite.config.ts new file mode 100644 index 00000000..9a130db0 --- /dev/null +++ b/extensions/bilibili/vite.config.ts @@ -0,0 +1,73 @@ +import { fileURLToPath } from 'node:url' +import path from 'node:path' +import { defineConfig, searchForWorkspaceRoot } from 'vite' +import vue from '@vitejs/plugin-vue' +import { federation } from '@module-federation/vite' + +import mfShared from '../mf-shared' +import extensionPackage from './package.json' +import { + createUiSourceAliases, + isPathInside, + resolveUiSourceForVite, + uiSourceDedupe, +} from '../../scripts/ui-source.mjs' + +const extensionRoot = fileURLToPath(new URL('.', import.meta.url)) +const extensionComponents = fileURLToPath(new URL('./src/components', import.meta.url)) + +export const bilibiliArtifactBase = './' +export const bilibiliBuildTarget = 'es2022' +export const bilibiliBuildOptions = { + target: bilibiliBuildTarget, + outDir: 'dist/web', + sourcemap: true, +} +export const bilibiliFederationOptions = { + name: 'extension.bilibili', + filename: 'remoteEntry.js', + manifest: true, + exposes: { '.': path.resolve(__dirname, './src/index.ts') }, + shared: mfShared( + extensionPackage.inkcre.module_federation.host_sdk_version, + extensionPackage.devDependencies['@inkcre/ui-web'] + ), +} + +export default defineConfig(async ({ command }) => { + const uiSource = await resolveUiSourceForVite(command) + const uiSourceComponents = uiSource ? path.resolve(uiSource.root, 'src/components') : null + + return { + plugins: [vue(), federation(bilibiliFederationOptions)], + resolve: { + alias: uiSource ? createUiSourceAliases(uiSource) : [], + ...(uiSource ? { dedupe: uiSourceDedupe } : {}), + }, + ...(uiSource + ? { + optimizeDeps: { exclude: ['@inkcre/ui-web'] }, + server: { + fs: { allow: [searchForWorkspaceRoot(extensionRoot), uiSource.root] }, + }, + } + : {}), + base: bilibiliArtifactBase, + build: bilibiliBuildOptions, + css: { + preprocessorOptions: { + scss: { + additionalData: (source: string, file: string) => { + if ( + isPathInside(extensionComponents, file) || + (uiSourceComponents && isPathInside(uiSourceComponents, file)) + ) { + return `@use "@inkcre/ui-web/styles/mixins" as *;@use "@inkcre/ui-web/styles/functions" as *;${source}` + } + return source + }, + }, + }, + }, + } +}) diff --git a/packages/core/package.json b/packages/core/package.json index 5d6b6e1d..aeab1dd8 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -9,6 +9,10 @@ "module": "./dist/index.js", "types": "./dist/index.d.ts", "exports": { + "./browser-automation/protocol": { + "types": "./dist/browser-automation-protocol.d.ts", + "import": "./dist/browser-automation-protocol.js" + }, ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" @@ -37,6 +41,7 @@ "dayjs": "^1.11.23", "htmlparser2": "10.1.0", "jose": "^5.0.0", + "puppeteer-core": "25.16.0", "semver": "7.8.5", "zod": "^4.6.7", "zod-class": "^0.0.18", diff --git a/packages/core/src/browser-automation/access.ts b/packages/core/src/browser-automation/access.ts new file mode 100644 index 00000000..ec15b2d9 --- /dev/null +++ b/packages/core/src/browser-automation/access.ts @@ -0,0 +1,296 @@ +import type { + Browser, + ConnectionTransport, + Page, +} from 'puppeteer-core/lib/puppeteer/puppeteer-core-browser.js' +import { configStore } from '../config' +import type { PeerRef } from '../peer/peer' +import { + BROWSER_AUTOMATION_PROTOCOL, + BrowserBridgeEventSchema, + BrowserBridgeResponseSchema, + BrowserProviderBindingSchema, + BrowserTabRefSchema, + type BrowserBridgeEvent, + type BrowserBridgeRequest, + type BrowserProviderBinding, + type BrowserTabRef, +} from './protocol' + +export type BrowserPage = Page +export type BrowserPageOptions = { + provider: PeerRef + target: { kind: 'new' } | { kind: 'existing'; tab: BrowserTabRef } + signal?: AbortSignal +} + +export class BrowserAutomationError extends Error { + override name = 'BrowserAutomationError' +} + +const requests = new Map< + string, + { binding: BrowserProviderBinding; resolve(value: unknown): void; reject(error: Error): void } +>() +const sessions = new Map() +const discoveries = new Set<(binding: BrowserProviderBinding) => void>() +let client: string | undefined + +function clientId(): string { + if (client) return client + if (typeof window === 'undefined') + throw new BrowserAutomationError('浏览器能力需要 Web 页面运行时。') + client = crypto.randomUUID() + window.addEventListener('message', receive) + return client +} + +function receive(event: MessageEvent): void { + if (event.source !== window || event.origin !== window.location.origin) return + const response = BrowserBridgeResponseSchema.safeParse(event.data) + if (response.success && response.data.client === client) { + const reply = response.data + const pending = requests.get(reply.request) + if (pending) { + const exactProvider = + pending.binding.provider === reply.provider && pending.binding.instance === reply.instance + if (!exactProvider) return + if (reply.ok) pending.resolve(reply.value) + else pending.reject(new BrowserAutomationError(reply.error ?? '浏览器操作失败。')) + } else if (reply.ok) { + const binding = BrowserProviderBindingSchema.safeParse(reply.value) + if (binding.success) for (const observe of discoveries) observe(binding.data) + } + return + } + const incoming = BrowserBridgeEventSchema.safeParse(event.data) + if (!incoming.success || incoming.data.client !== client) return + sessions.get(incoming.data.session)?.receive(incoming.data) +} + +function bindingKey(): string { + const { INKCRE_PEER_ID, INKCRE_PGREST_URL } = configStore.metaConfig + if (!INKCRE_PEER_ID || !INKCRE_PGREST_URL) + throw new BrowserAutomationError('请先连接信息库，再配置本 Peer 的浏览器绑定。') + return `inkcre.browser-bindings.v1:${INKCRE_PGREST_URL}:${INKCRE_PEER_ID}` +} + +function bindingFor(provider: PeerRef): BrowserProviderBinding { + const binding = BrowserAccess.bindings().find((item) => item.provider === provider) + if (!binding) throw new BrowserAutomationError('本 Web Peer 尚未绑定所选浏览器。') + return binding +} + +function request( + binding: BrowserProviderBinding, + operation: BrowserBridgeRequest['operation'], + parameters: Record = {}, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const id = crypto.randomUUID() + const pageClient = clientId() + return new Promise((resolve, reject) => { + const finish = (error?: Error, value?: unknown) => { + clearTimeout(timeout) + signal?.removeEventListener('abort', abort) + requests.delete(id) + if (error) reject(error) + else resolve(value) + } + const abort = () => finish(new BrowserAutomationError('浏览器操作已取消。')) + const timeout = setTimeout( + () => + finish( + new BrowserAutomationError('浏览器桥未响应；已发送操作的结果可能未知，请勿自动重放。') + ), + 15_000 + ) + requests.set(id, { + binding, + resolve: (value) => finish(undefined, value), + reject: (error) => finish(error), + }) + signal?.addEventListener('abort', abort, { once: true }) + window.postMessage( + { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'request', + client: pageClient, + request: id, + provider: binding.provider, + instance: binding.instance, + operation, + parameters, + } satisfies BrowserBridgeRequest, + window.location.origin + ) + }) +} + +class RelayTransport implements ConnectionTransport { + onmessage?: (message: string) => void + onclose?: () => void + private closed = false + + constructor( + readonly binding: BrowserProviderBinding, + readonly session: string + ) { + sessions.set(session, this) + } + + send(message: string): void { + if (this.closed) throw new BrowserAutomationError('此浏览器会话已经结束。') + void request(this.binding, 'cdp', { session: this.session, message }).catch(() => this.close()) + } + + receive(event: BrowserBridgeEvent): void { + if ( + this.closed || + event.provider !== this.binding.provider || + event.instance !== this.binding.instance + ) + return + if (event.event === 'closed') this.close() + else if (event.data !== undefined) this.onmessage?.(event.data) + } + + close(): void { + if (this.closed) return + this.closed = true + sessions.delete(this.session) + this.onclose?.() + } +} + +/** Local binding is an execution scope, independent of current browser connectivity or login. */ +export class BrowserAccess { + static bindings(): BrowserProviderBinding[] { + try { + const raw: unknown = JSON.parse(localStorage.getItem(bindingKey()) ?? '[]') + return BrowserProviderBindingSchema.array().parse(raw) + } catch { + return [] + } + } + + static hasBinding(provider: PeerRef): boolean { + return this.bindings().some((item) => item.provider === provider && item.protocol === 1) + } + + static bind(binding: BrowserProviderBinding): void { + const parsed = BrowserProviderBindingSchema.parse(binding) + localStorage.setItem( + bindingKey(), + JSON.stringify([ + ...this.bindings().filter((item) => item.provider !== parsed.provider), + parsed, + ]) + ) + } + + static unbind(provider: PeerRef): void { + localStorage.setItem( + bindingKey(), + JSON.stringify(this.bindings().filter((item) => item.provider !== provider)) + ) + } + + /** Discover only extensions whose settings explicitly authorized this exact Web origin. */ + static async discover(signal?: AbortSignal): Promise { + signal?.throwIfAborted() + const found = new Map() + const pageClient = clientId() + const observe = (binding: BrowserProviderBinding) => found.set(binding.provider, binding) + discoveries.add(observe) + try { + window.postMessage( + { + protocol: BROWSER_AUTOMATION_PROTOCOL, + kind: 'request', + client: pageClient, + request: crypto.randomUUID(), + operation: 'discover', + parameters: {}, + } satisfies BrowserBridgeRequest, + window.location.origin + ) + await new Promise((resolve, reject) => { + const abort = () => { + clearTimeout(timer) + reject(signal?.reason) + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', abort) + resolve() + }, 1000) + signal?.addEventListener('abort', abort, { once: true }) + }) + return [...found.values()] + } finally { + discoveries.delete(observe) + } + } + + static async tabs(provider: PeerRef, signal?: AbortSignal): Promise { + return BrowserTabRefSchema.array().parse( + await request(bindingFor(provider), 'tabs', {}, signal) + ) + } + + static async withPage( + options: BrowserPageOptions, + action: (page: Page) => Promise + ): Promise { + options.signal?.throwIfAborted() + const binding = bindingFor(options.provider) + // Opening is not cancellable midway: learn the session identity before releasing it. + const session = crypto.randomUUID() + try { + await request(binding, 'open', { session, target: options.target }) + } catch (error) { + await request(binding, 'close', { session }).catch(() => undefined) + throw error + } + const transport = new RelayTransport(binding, session) + let releasing: Promise | undefined + const release = () => { + releasing ??= request(binding, 'close', { session }) + return releasing + } + const abort = () => { + transport.close() + void release().catch(() => undefined) + } + options.signal?.addEventListener('abort', abort, { once: true }) + let browser: Browser | undefined + try { + options.signal?.throwIfAborted() + // The pinned CDP entry avoids the browser entry's Node channel/BiDi import graph. + const { _connectToCdpBrowser } = + await import('puppeteer-core/internal/cdp/BrowserConnector.js') + browser = await _connectToCdpBrowser( + transport, + '', + { defaultViewport: null, protocolTimeout: 30_000 }, + () => undefined + ) + const [page] = await browser.pages() + if (!page) throw new BrowserAutomationError('浏览器没有返回可控制页面。') + options.signal?.throwIfAborted() + const result = await action(page) + options.signal?.throwIfAborted() + return result + } finally { + options.signal?.removeEventListener('abort', abort) + // Await the callback before cleanup; an abort does not roll back already dispatched effects. + try { + await release() + } finally { + transport.close() + await browser?.disconnect() + } + } + } +} diff --git a/packages/core/src/browser-automation/index.ts b/packages/core/src/browser-automation/index.ts new file mode 100644 index 00000000..d0a5e745 --- /dev/null +++ b/packages/core/src/browser-automation/index.ts @@ -0,0 +1,3 @@ +export { BrowserAccess, BrowserAutomationError } from './access' +export type { BrowserPage, BrowserPageOptions } from './access' +export * from './protocol' diff --git a/packages/core/src/browser-automation/protocol.ts b/packages/core/src/browser-automation/protocol.ts new file mode 100644 index 00000000..9087fb3e --- /dev/null +++ b/packages/core/src/browser-automation/protocol.ts @@ -0,0 +1,61 @@ +import { z } from 'zod' + +export const BROWSER_AUTOMATION_PROTOCOL = 'inkcre.browser.v1' +export const BROWSER_AUTOMATION_PORT = 'inkcre-browser-v1' + +export const BrowserProviderBindingSchema = z.object({ + provider: z.uuid(), + instance: z.uuid(), + name: z.string(), + extensionVersion: z.string(), + protocol: z.literal(1), +}) +export type BrowserProviderBinding = z.infer + +export const BrowserTabRefSchema = z.object({ + provider: z.uuid(), + instance: z.uuid(), + epoch: z.uuid(), + reference: z.uuid(), + tabId: z.number().int().nonnegative(), + title: z.string().optional(), + url: z.string().optional(), +}) +export type BrowserTabRef = z.infer + +export const BrowserBridgeRequestSchema = z.object({ + protocol: z.literal(BROWSER_AUTOMATION_PROTOCOL), + kind: z.literal('request'), + client: z.uuid(), + request: z.uuid(), + provider: z.uuid().optional(), + instance: z.uuid().optional(), + operation: z.enum(['discover', 'tabs', 'open', 'cdp', 'close']), + parameters: z.record(z.string(), z.unknown()).default({}), +}) +export type BrowserBridgeRequest = z.infer + +export const BrowserBridgeResponseSchema = z.object({ + protocol: z.literal(BROWSER_AUTOMATION_PROTOCOL), + kind: z.literal('response'), + client: z.uuid(), + request: z.uuid(), + provider: z.uuid(), + instance: z.uuid(), + ok: z.boolean(), + value: z.unknown().optional(), + error: z.string().optional(), +}) +export type BrowserBridgeResponse = z.infer + +export const BrowserBridgeEventSchema = z.object({ + protocol: z.literal(BROWSER_AUTOMATION_PROTOCOL), + kind: z.literal('event'), + client: z.uuid(), + provider: z.uuid(), + instance: z.uuid(), + session: z.uuid(), + event: z.enum(['message', 'closed']), + data: z.string().optional(), +}) +export type BrowserBridgeEvent = z.infer diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index bcfefaaf..adb5f08f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -10,6 +10,7 @@ // Core singleton export { store } from './store' +export * from './browser-automation' // Base layer export { APIError, DBAPIClient, rawPostgrestFetch } from './base' @@ -117,6 +118,8 @@ export { SourceForm, SourceType, SourceManager, + SourceResolver, + SourceAnchorSchema, SOURCE_COLLECT_JOB_TYPE, SOURCE_BACKFILL_JOB_TYPE, type SourceImplementation, diff --git a/packages/core/src/info-base/resolvers/contracts.ts b/packages/core/src/info-base/resolvers/contracts.ts index 150d36f3..26effdb2 100644 --- a/packages/core/src/info-base/resolvers/contracts.ts +++ b/packages/core/src/info-base/resolvers/contracts.ts @@ -9,6 +9,7 @@ export const CORE_RESOLVER_IDS = [ 'core.epub.v1', 'core.zip.v1', 'core.file.v1', + 'core.source.v1', ] as const export type CoreResolverId = (typeof CORE_RESOLVER_IDS)[number] diff --git a/packages/core/src/info-base/resolvers/index.ts b/packages/core/src/info-base/resolvers/index.ts index 66cafa38..9c93e373 100644 --- a/packages/core/src/info-base/resolvers/index.ts +++ b/packages/core/src/info-base/resolvers/index.ts @@ -47,6 +47,7 @@ import { TextResolver } from './text' import { MarkdownResolver } from './markdown' import { VideoResolver } from './video' import { ZipResolver } from './zip' +import { SourceResolver } from '../../source/resolver' export function registerCoreResolvers(): void { for (const resolverClass of [ @@ -60,6 +61,7 @@ export function registerCoreResolvers(): void { EpubResolver, ZipResolver, FileResolver, + SourceResolver, ]) { Resolver.register(resolverClass.type, resolverClass) } diff --git a/packages/core/src/job/manager.ts b/packages/core/src/job/manager.ts index e6fe5af9..d86ce9b0 100644 --- a/packages/core/src/job/manager.ts +++ b/packages/core/src/job/manager.ts @@ -118,6 +118,12 @@ export class JobManager { private static async runAdmitted(id: JobRef): Promise { const candidate = await Job.get(id) if (!this.accepting || candidate.status !== JobStatus.PENDING) return false + if (candidate.abort_requested) { + const claimed = await this.claim(candidate) + if (!claimed) return false + await this.observeExecution(claimed, () => this.close(claimed, JobStatus.ABORTED)) + return true + } let prepared: [JobHandler, unknown] | null try { prepared = await this.prepare(candidate) @@ -126,7 +132,7 @@ export class JobManager { if (!this.accepting) return false const claimed = await this.claim(candidate) if (!claimed) return false - if (!this.accepting) { + if (!this.accepting || claimed.abort_requested) { await this.observeExecution(claimed, () => this.close(claimed, JobStatus.ABORTED)) return true } @@ -140,7 +146,7 @@ export class JobManager { if (!claimed) return false // stopWorker may have run while claim was in flight. Do not start a handler // after its Extension resources have been disposed. - if (!this.accepting) { + if (!this.accepting || claimed.abort_requested) { await this.observeExecution(claimed, () => this.close(claimed, JobStatus.ABORTED)) return true } diff --git a/packages/core/src/source/index.ts b/packages/core/src/source/index.ts index 1e77ecd0..7f0f5e7d 100644 --- a/packages/core/src/source/index.ts +++ b/packages/core/src/source/index.ts @@ -1,3 +1,4 @@ export * from './source-type' export * from './source' export * from './manager' +export * from './resolver' diff --git a/packages/core/src/source/manager.ts b/packages/core/src/source/manager.ts index de99ffbd..e2f0cc64 100644 --- a/packages/core/src/source/manager.ts +++ b/packages/core/src/source/manager.ts @@ -16,6 +16,8 @@ export interface SourceImplementation< > { collectConfig: CollectConfig backfillConfig: BackfillConfig + canCollect?(source: Source, config: z.output): boolean | Promise + canBackfill?(source: Source, config: unknown): boolean | Promise collect( source: Source, job: Job, @@ -42,7 +44,8 @@ export class SourceManager { this.implementations.set(type, implementation) } - static unregister(type: string): void { + static unregister(type: string, implementation?: SourceImplementation): void { + if (implementation && this.implementations.get(type) !== implementation) return this.implementations.delete(type) } @@ -60,7 +63,13 @@ export class SourceManager { JobManager.registerHandler(SOURCE_COLLECT_JOB_TYPE, { parameters: SourceCommandParameters, async canHandle(parameters) { - return (await SourceManager.resolve(parameters.source)) !== null + const resolved = await SourceManager.resolve(parameters.source) + if (!resolved) return false + const [source, implementation] = resolved + const config = implementation.collectConfig.safeParse(parameters.config) + // Invalid commands must be claimed and fail in handle, rather than remain pending forever. + if (!config.success) return true + return implementation.canCollect ? implementation.canCollect(source, config.data) : true }, async handle(job, parameters, signal) { const resolved = await SourceManager.resolve(parameters.source) @@ -75,7 +84,12 @@ JobManager.registerHandler(SOURCE_BACKFILL_JOB_TYPE, { parameters: SourceCommandParameters, async canHandle(parameters) { const resolved = await SourceManager.resolve(parameters.source) - return resolved !== null && resolved[1].backfillConfig !== null && !!resolved[1].backfill + if (!resolved) return false + const [source, implementation] = resolved + if (!implementation.backfillConfig || !implementation.backfill) return false + const config = implementation.backfillConfig.safeParse(parameters.config) + if (!config.success) return true + return implementation.canBackfill ? implementation.canBackfill(source, config.data) : true }, async handle(job, parameters, signal) { const resolved = await SourceManager.resolve(parameters.source) diff --git a/packages/core/src/source/resolver.ts b/packages/core/src/source/resolver.ts new file mode 100644 index 00000000..8e5b7dd1 --- /dev/null +++ b/packages/core/src/source/resolver.ts @@ -0,0 +1,38 @@ +import { z } from 'zod' +import { Resolver } from '../info-base/resolvers/base' +import { ResolverContentError, type ProjectionOptions } from '../info-base/resolvers/contracts' +import { decodeUnicode } from '../info-base/resolvers/decode' + +export const SourceAnchorSchema = z.strictObject({ + id: z.number().int(), + type: z.string(), + nickname: z.string().nullable().default(null), +}) + +/** The Source anchor describes collection provenance; its outgoing relations lead to business data. */ +export class SourceResolver extends Resolver { + static readonly type = 'core.source.v1' + + protected async _getSolvedContent(options: ProjectionOptions): Promise { + const raw = decodeUnicode(await this.getRawContent(options), SourceResolver.type) + let decoded: unknown + try { + decoded = JSON.parse(raw) + } catch { + throw new ResolverContentError(SourceResolver.type, 'expected a JSON Source anchor') + } + const parsed = SourceAnchorSchema.safeParse(decoded) + if (!parsed.success) + throw new ResolverContentError( + SourceResolver.type, + 'expected Source id, type and optional nickname' + ) + const source = parsed.data + const name = source.nickname ? ` named ${source.nickname}` : '' + return `Source ${source.id}${name} uses ${source.type}.` + } + + async getText(options: ProjectionOptions = {}): Promise { + return this.getSolvedContent(options) + } +} diff --git a/packages/core/tsdown.config.ts b/packages/core/tsdown.config.ts index 1ca641c0..a632136a 100644 --- a/packages/core/tsdown.config.ts +++ b/packages/core/tsdown.config.ts @@ -12,6 +12,14 @@ const common = { // Separate graphs keep the SDK's declarations independent of Vue peer dependencies. export default defineConfig([ - { ...common, entry: ['src/index.ts'], outDir: 'dist', clean: false }, + { + ...common, + entry: { + index: 'src/index.ts', + 'browser-automation-protocol': 'src/browser-automation/protocol.ts', + }, + outDir: 'dist', + clean: false, + }, { ...common, entry: ['src/sdk.ts'], outDir: 'dist/sdk', clean: false }, ]) diff --git a/patches/puppeteer-core@25.16.0.patch b/patches/puppeteer-core@25.16.0.patch new file mode 100644 index 00000000..b6f1a2ce --- /dev/null +++ b/patches/puppeteer-core@25.16.0.patch @@ -0,0 +1,51 @@ +diff --git a/src/cdp/ExtensionTransport.ts b/src/cdp/ExtensionTransport.ts +--- a/src/cdp/ExtensionTransport.ts ++++ b/src/cdp/ExtensionTransport.ts +@@ -190,8 +190,8 @@ + }); + } + +- close(): void { ++ close(): Promise { + chrome.debugger.onEvent.removeListener(this.#debuggerEventHandler); +- void chrome.debugger.detach({tabId: this.#tabId}); ++ return chrome.debugger.detach({tabId: this.#tabId}); + } + } +diff --git a/lib/puppeteer/cdp/ExtensionTransport.js b/lib/puppeteer/cdp/ExtensionTransport.js +--- a/lib/puppeteer/cdp/ExtensionTransport.js ++++ b/lib/puppeteer/cdp/ExtensionTransport.js +@@ -169,7 +169,7 @@ + } + close() { + chrome.debugger.onEvent.removeListener(this.#debuggerEventHandler); +- void chrome.debugger.detach({ tabId: this.#tabId }); ++ return chrome.debugger.detach({ tabId: this.#tabId }); + } + } + //# sourceMappingURL=ExtensionTransport.js.map +\ No newline at end of file +diff --git a/lib/puppeteer/cdp/ExtensionTransport.d.ts b/lib/puppeteer/cdp/ExtensionTransport.d.ts +--- a/lib/puppeteer/cdp/ExtensionTransport.d.ts ++++ b/lib/puppeteer/cdp/ExtensionTransport.d.ts +@@ -23,6 +23,6 @@ + */ + constructor(tabId: number); + send(message: string): void; +- close(): void; ++ close(): Promise; + } + //# sourceMappingURL=ExtensionTransport.d.ts.map +\ No newline at end of file +diff --git a/lib/types.d.ts b/lib/types.d.ts +--- a/lib/types.d.ts ++++ b/lib/types.d.ts +@@ -3102,7 +3102,7 @@ + onmessage?: (message: string) => void; + onclose?: () => void; + send(message: string): void; +- close(): void; ++ close(): Promise; + } + + /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09ad34fb..d3d463a8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,7 @@ overrides: patchedDependencies: '@inkcre/ui-web@2.5.0': 2b0f967ae08c3559202523c4ccb3ed4769ff9d8863a52c600259231171d56051 + puppeteer-core@25.16.0: ca9d8d4e7430e6c3ff5cf5f5802f5aa087cc157c8a5d3c9547a27426b22d8316 importers: @@ -227,6 +228,9 @@ importers: '@mozilla/readability': specifier: ^0.6.0 version: 0.6.0 + puppeteer-core: + specifier: 25.16.0 + version: 25.16.0(patch_hash=ca9d8d4e7430e6c3ff5cf5f5802f5aa087cc157c8a5d3c9547a27426b22d8316) vue: specifier: ^3.5.44 version: 3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2) @@ -253,6 +257,55 @@ importers: specifier: ^0.20.6 version: 0.20.27(@rspack/core@2.1.5)(@types/node@22.17.2)(eslint@9.39.5(jiti@2.7.0))(jiti@2.7.0)(lightningcss@1.33.0)(oxc-parser@0.131.0)(rolldown@1.2.0)(rollup@4.59.0)(sass-embedded@1.90.0)(sass@1.101.6)(yaml@2.9.0) + extensions/bilibili: + dependencies: + '@inkcre/core': + specifier: workspace:* + version: link:../../packages/core + '@vueuse/core': + specifier: ^14.0.0 + version: 14.4.0(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + pinia: + specifier: ^3.0.0 + version: 3.0.4(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + vue: + specifier: ^3.5.25 + version: 3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2) + vue-router: + specifier: ^4.5.0 + version: 4.6.4(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + devDependencies: + '@inkcre/ext-dev-utils': + specifier: workspace:* + version: link:../../packages/ext-dev-utils + '@inkcre/ui-web': + specifier: https://github.com/InKCre/ui/releases/download/%40inkcre%2Fui-web%402.7.0/inkcre-ui-web-2.7.0.tgz + version: https://github.com/InKCre/ui/releases/download/%40inkcre%2Fui-web%402.7.0/inkcre-ui-web-2.7.0.tgz(unocss@66.7.5(vite@7.3.6(@types/node@22.17.2)(jiti@2.7.0)(lightningcss@1.33.0)(sass-embedded@1.90.0)(sass@1.101.6)(yaml@2.9.0)))(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + '@module-federation/vite': + specifier: ^1.1.0 + version: 1.7.1(rollup@4.59.0) + '@vitejs/plugin-vue': + specifier: ^6.0.1 + version: 6.0.3(vite@7.3.6(@types/node@22.17.2)(jiti@2.7.0)(lightningcss@1.33.0)(sass-embedded@1.90.0)(sass@1.101.6)(yaml@2.9.0))(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + '@vue/tsconfig': + specifier: 'catalog:' + version: 0.9.1(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)(vue@3.5.44(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2)) + sass-embedded: + specifier: ^1.90.0 + version: 1.90.0 + typescript: + specifier: npm:typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2 + version: typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2 + vite: + specifier: ^7.0.6 + version: 7.3.6(@types/node@22.17.2)(jiti@2.7.0)(lightningcss@1.33.0)(sass-embedded@1.90.0)(sass@1.101.6)(yaml@2.9.0) + vue-tsc: + specifier: 'catalog:' + version: 3.3.8(typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2) + zod: + specifier: ^4.6.7 + version: 4.6.7 + extensions/caldav: dependencies: '@inkcre/core': @@ -646,6 +699,9 @@ importers: jose: specifier: ^5.0.0 version: 5.10.0 + puppeteer-core: + specifier: 25.16.0 + version: 25.16.0(patch_hash=ca9d8d4e7430e6c3ff5cf5f5802f5aa087cc157c8a5d3c9547a27426b22d8316) semver: specifier: 7.8.5 version: 7.8.5 @@ -2327,6 +2383,19 @@ packages: '@poppinss/exception@1.2.3': resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + '@puppeteer/browsers@3.2.6': + resolution: {integrity: sha512-MJP29gVoHznmqmwhH1tlZqzfeQ3aylTpSSa6WKZXtkwYUH02i9imBmWGTyZrbnAou5paX78SE4jMAUqnEPf8ug==} + engines: {node: '>=22.12.0'} + hasBin: true + peerDependencies: + proxy-agent: '>=8.0.1' + yauzl: ^2.10.0 || ^3.4.0 + peerDependenciesMeta: + proxy-agent: + optional: true + yauzl: + optional: true + '@quansync/fs@1.0.0': resolution: {integrity: sha512-4TJ3DFtlf1L5LDMaM6CanJ/0lckGNtJcMjQ1NAV6zDmA0tEHKZtxNKin8EgPaVX1YzljbxckyT2tJrpQKAtngQ==} @@ -3510,6 +3579,12 @@ packages: engines: {node: '>=12.13.0'} hasBin: true + chromium-bidi@157.0.8090-0: + resolution: {integrity: sha512-KJw7IQgqXwxpJgJlEIPg5mr2ZoqA+jkTRtc5EABI7sUJ3bu3cpI7NuEBoj4Gn1AsF78CaT9So5WQWspK3/wYag==} + engines: {node: '>=20.19.0 <22.0.0 || >=22.12.0'} + peerDependencies: + devtools-protocol: '*' + ci-info@4.4.0: resolution: {integrity: sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==} engines: {node: '>=8'} @@ -3533,6 +3608,10 @@ packages: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} + cliui@9.0.1: + resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} + engines: {node: '>=20'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -3751,6 +3830,9 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + devtools-protocol@0.0.1704951: + resolution: {integrity: sha512-GQhQOs9u/Mvm1zMtJIH872JS4AskuZdUSsPNPk4ChahtWAUqgz6atEtIsMt+nO7MqF1Ic4HxII/k8Cu36JivNg==} + dom-serializer@2.0.0: resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} @@ -4578,6 +4660,10 @@ packages: mlly@1.8.2: resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + modern-tar@0.8.6: + resolution: {integrity: sha512-LYSks0tNVqbtez/9OB0YFP6qLRSqkdAxJWOQpVaR67lNTbj2zNLyYpowkRMia5lXLCZKzxwoxkxH2MbbBR2aEg==} + engines: {node: '>=18.0.0'} + mrmime@2.0.1: resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==} engines: {node: '>=10'} @@ -5054,6 +5140,10 @@ packages: resolution: {integrity: sha512-LjgDO2zPtoXP2wJpDjZrGdojii1uqO0cnwKoIoUzkfS98HDmbeiGmYiXo3lXeFlq2xvne1QFQhwYXSUCLKtEuA==} engines: {node: '>=12.20'} + puppeteer-core@25.16.0: + resolution: {integrity: sha512-VA5Pf54N8shfopNkZZXCdiiwfnnY8ZJYAMqFQRhlH5mGUhfwf53Ar770GPtdpTDLUqvQeYpIBtC9/G2Jij2IJw==} + engines: {node: '>=22.12.0'} + quansync@0.2.11: resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} @@ -5574,6 +5664,9 @@ packages: type-level-regexp@0.1.17: resolution: {integrity: sha512-wTk4DH3cxwk196uGLK/E9pE45aLfeKJacKmcEgEOA/q5dnPGNxXt0cfYdFxb57L+sEpf1oJH4Dnx/pnRcku9jg==} + typed-query-selector@2.12.3: + resolution: {integrity: sha512-/dtuMZyutHLlrxWhhC1uib00ELUr2dZ8C9B8YzZYsJmeE3V56OxRblrSM2UaF142pcMtak7ehZ6+poWGeehNIg==} + typedarray@0.0.6: resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} @@ -5893,6 +5986,9 @@ packages: resolution: {integrity: sha512-rQicL7OwuqWdQWI33JkSXKcp7cuv1mJG8u3jRQwx/8aDsmhbTHs9ZRmNYOL+LX0wX8edIEQX8jj4bB60GoXtKA==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} + webdriver-bidi-protocol@1.0.1: + resolution: {integrity: sha512-WXWs/aXF2a6Al1Z2/V/7Gp2oPZOAAmHjJOTmv4KCXRJ122IMj3tSSMc6a4q3K4QOmhQbse/Ahr+oTVJOU2dUiw==} + webpack-virtual-modules@0.6.2: resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} @@ -5966,6 +6062,18 @@ packages: utf-8-validate: optional: true + ws@8.22.1: + resolution: {integrity: sha512-oyL6OPYme/CmiFvNuGzeoxCJ4nOrZE2O1FwM9MWM1AsGri6nLXB2pg+lNA8aLeXMSav0J6eIVS1j6vUPzUtK9g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + wsl-utils@0.1.0: resolution: {integrity: sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==} engines: {node: '>=18'} @@ -6012,6 +6120,10 @@ packages: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} + yargs-parser@22.0.0: + resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yargs@17.7.2: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} @@ -6020,6 +6132,10 @@ packages: resolution: {integrity: sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==} engines: {node: '>=12'} + yargs@18.2.0: + resolution: {integrity: sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -7596,6 +7712,11 @@ snapshots: '@poppinss/exception@1.2.3': {} + '@puppeteer/browsers@3.2.6': + dependencies: + modern-tar: 0.8.6 + yargs: 18.2.0 + '@quansync/fs@1.0.0': dependencies: quansync: 1.0.0 @@ -8676,6 +8797,12 @@ snapshots: transitivePeerDependencies: - supports-color + chromium-bidi@157.0.8090-0(devtools-protocol@0.0.1704951): + dependencies: + devtools-protocol: 0.0.1704951 + mitt: 3.0.1 + zod: 4.6.7 + ci-info@4.4.0: {} citty@0.2.2: {} @@ -8697,6 +8824,12 @@ snapshots: strip-ansi: 6.0.1 wrap-ansi: 7.0.0 + cliui@9.0.1: + dependencies: + string-width: 7.2.0 + strip-ansi: 7.2.0 + wrap-ansi: 9.0.2 + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -8880,6 +9013,8 @@ snapshots: detect-libc@2.1.2: {} + devtools-protocol@0.0.1704951: {} + dom-serializer@2.0.0: dependencies: domelementtype: 2.3.0 @@ -9676,6 +9811,8 @@ snapshots: pkg-types: 1.3.1 ufo: 1.6.4 + modern-tar@0.8.6: {} + mrmime@2.0.1: {} ms@2.1.3: {} @@ -10064,6 +10201,20 @@ snapshots: dependencies: escape-goat: 4.0.0 + puppeteer-core@25.16.0(patch_hash=ca9d8d4e7430e6c3ff5cf5f5802f5aa087cc157c8a5d3c9547a27426b22d8316): + dependencies: + '@puppeteer/browsers': 3.2.6 + chromium-bidi: 157.0.8090-0(devtools-protocol@0.0.1704951) + devtools-protocol: 0.0.1704951 + typed-query-selector: 2.12.3 + webdriver-bidi-protocol: 1.0.1 + ws: 8.22.1 + transitivePeerDependencies: + - bufferutil + - proxy-agent + - utf-8-validate + - yauzl + quansync@0.2.11: {} quansync@1.0.0: {} @@ -10599,6 +10750,8 @@ snapshots: type-level-regexp@0.1.17: {} + typed-query-selector@2.12.3: {} + typedarray@0.0.6: {} typescript-native-bridge@6.0.3-bridge.19.tsgo.7.0.2: @@ -10968,6 +11121,8 @@ snapshots: transitivePeerDependencies: - supports-color + webdriver-bidi-protocol@1.0.1: {} + webpack-virtual-modules@0.6.2: {} when-exit@2.1.5: {} @@ -11034,12 +11189,14 @@ snapshots: wrap-ansi@9.0.2: dependencies: - ansi-styles: 6.2.1 + ansi-styles: 6.2.3 string-width: 7.2.0 strip-ansi: 7.2.0 ws@8.21.0: {} + ws@8.22.1: {} + wsl-utils@0.1.0: dependencies: is-wsl: 3.1.0 @@ -11136,6 +11293,8 @@ snapshots: yargs-parser@21.1.1: {} + yargs-parser@22.0.0: {} + yargs@17.7.2: dependencies: cliui: 8.0.1 @@ -11156,6 +11315,15 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + yargs@18.2.0: + dependencies: + cliui: 9.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + string-width: 8.2.2 + y18n: 5.0.8 + yargs-parser: 22.0.0 + yocto-queue@0.1.0: optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5f0ff9cd..d4d46a9e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -57,5 +57,7 @@ minimumReleaseAgeExclude: - marked@18.1.2 patchedDependencies: + # shortcut: 上游 ExtensionTransport.close 尚未返回 detach Promise；上游修复后移除此补丁。 + puppeteer-core@25.16.0: patches/puppeteer-core@25.16.0.patch # shortcut: 修复剩余 2.5.0 消费者重复渲染输入值和 ID；升级到上游修复版时移除对应补丁。 '@inkcre/ui-web@2.5.0': patches/@inkcre__ui-web@2.5.0.patch diff --git a/scripts/check-package-contract.mjs b/scripts/check-package-contract.mjs index 55f20aad..84b6af61 100644 --- a/scripts/check-package-contract.mjs +++ b/scripts/check-package-contract.mjs @@ -38,6 +38,7 @@ await Promise.all( 'apps/webext/.output/chrome-mv3/options.html', 'apps/webext/.output/chrome-mv3/popup.html', 'extensions/github/dist/web/remoteEntry.js', + 'extensions/bilibili/dist/web/remoteEntry.js', 'extensions/caldav/dist/web/remoteEntry.js', 'extensions/capture/dist/web/remoteEntry.js', 'extensions/local-files/dist/web/remoteEntry.js', diff --git a/scripts/check-workspace-contract.mjs b/scripts/check-workspace-contract.mjs index 917d94c8..fbb71f28 100644 --- a/scripts/check-workspace-contract.mjs +++ b/scripts/check-workspace-contract.mjs @@ -6,6 +6,12 @@ const repoRoot = fileURLToPath(new URL('..', import.meta.url)) const jsonOutput = process.argv.includes('--json') const workspaces = [ + { + path: 'extensions/bilibili', + name: '@inkcre/ext-bilibili', + requiredScripts: ['build', 'dev', 'type-check'], + builder: 'vite', + }, { path: 'extensions/local-files', name: '@inkcre/ext-local-files', diff --git a/scripts/test-webext-e2e.mjs b/scripts/test-webext-e2e.mjs index 1f6100d2..de78ba4a 100644 --- a/scripts/test-webext-e2e.mjs +++ b/scripts/test-webext-e2e.mjs @@ -13,4 +13,11 @@ function run(command, args) { run('pnpm', ['--filter', '@inkcre/core', 'build']) run('pnpm', ['--filter', '@inkcre/webext', 'build']) +run('node', [ + '--test', + 'scripts/tests/browser-automation-transport.test.mjs', + 'scripts/tests/job-cancellation.test.mjs', + 'scripts/tests/source-anchor.test.mjs', +]) +run('node', ['scripts/verify-browser-automation.mjs']) run('pnpm', ['exec', 'playwright', 'test', '--project', 'browser-extension']) diff --git a/scripts/tests/browser-automation-transport.test.mjs b/scripts/tests/browser-automation-transport.test.mjs new file mode 100644 index 00000000..8b29ed59 --- /dev/null +++ b/scripts/tests/browser-automation-transport.test.mjs @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict' +import { createRequire } from 'node:module' +import { pathToFileURL } from 'node:url' +import test from 'node:test' + +const require = createRequire(new URL('../../packages/core/package.json', import.meta.url)) +const { ExtensionTransport } = await import( + pathToFileURL(require.resolve('puppeteer-core/internal/cdp/ExtensionTransport.js')).href +) + +void test('ExtensionTransport close awaits native detach and removes its event listener', async () => { + const original = globalThis.chrome + const listeners = new Set() + let completeDetach + const detached = new Promise((resolve) => { + completeDetach = resolve + }) + const calls = [] + globalThis.chrome = { + debugger: { + attach: async (target) => calls.push(['attach', target]), + detach: (target) => { + calls.push(['detach', target]) + return detached + }, + onEvent: { + addListener: (listener) => listeners.add(listener), + removeListener: (listener) => listeners.delete(listener), + }, + }, + } + try { + const transport = await ExtensionTransport.connectTab(42) + assert.equal(listeners.size, 1) + let finished = false + const closing = transport.close() + assert.ok(closing instanceof Promise) + void closing.then(() => { + finished = true + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(finished, false) + assert.equal(listeners.size, 0) + completeDetach() + await closing + assert.equal(finished, true) + assert.deepEqual(calls, [ + ['attach', { tabId: 42 }], + ['detach', { tabId: 42 }], + ]) + } finally { + globalThis.chrome = original + } +}) + +void test('ExtensionTransport close exposes detach failure to the session owner', async () => { + const original = globalThis.chrome + const listeners = new Set() + globalThis.chrome = { + debugger: { + attach: async () => undefined, + detach: async () => { + throw new Error('native detach failed') + }, + onEvent: { + addListener: (listener) => listeners.add(listener), + removeListener: (listener) => listeners.delete(listener), + }, + }, + } + try { + const transport = await ExtensionTransport.connectTab(1) + await assert.rejects(transport.close(), /native detach failed/) + assert.equal(listeners.size, 0) + } finally { + globalThis.chrome = original + } +}) diff --git a/scripts/tests/browser-automation/index.html b/scripts/tests/browser-automation/index.html new file mode 100644 index 00000000..1301179a --- /dev/null +++ b/scripts/tests/browser-automation/index.html @@ -0,0 +1,11 @@ + + + + + Browser automation acceptance + + +

Running browser automation acceptance…

+ + + diff --git a/scripts/tests/browser-automation/main.ts b/scripts/tests/browser-automation/main.ts new file mode 100644 index 00000000..efd45124 --- /dev/null +++ b/scripts/tests/browser-automation/main.ts @@ -0,0 +1,250 @@ +import { BrowserAccess, configStore, MetaConfigSchema } from '@inkcre/core' + +let stage = 'discover' + +function check(value: unknown, message: string): asserts value { + if (!value) throw new Error(message) +} + +async function main() { + if (location.pathname === '/unpaired' || location.pathname === '/paired') { + const found = await BrowserAccess.discover() + await fetch('/probe-report', { + method: 'POST', + body: JSON.stringify({ + token: new URLSearchParams(location.search).get('token'), + count: found.length, + providers: found.map((item) => item.provider), + }), + }) + return + } + const config = await fetch('/config').then((response) => response.json()) + configStore.metaConfig = MetaConfigSchema.parse({ + INKCRE_PGREST_URL: `${location.origin}/unused-database`, + INKCRE_PEER_ID: crypto.randomUUID(), + }) + const found = await BrowserAccess.discover() + check(found.length === 2, `Expected two authorized providers, found ${found.length}`) + const binding = found.find((item) => item.provider === config.provider)! + check(binding.provider === config.provider, 'Discovery returned another provider') + BrowserAccess.bind(binding) + check(BrowserAccess.hasBinding(binding.provider), 'Explicit binding was not persisted') + const baseline = await BrowserAccess.tabs(binding.provider) + let stalePage: { evaluate: (value: string) => Promise } | undefined + const extracted = await BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'new' } }, + async (page) => { + stalePage = page + stage = 'goto' + await page.goto(`${location.origin}/target`) + stage = 'wait-selector' + await page.waitForSelector('#more') + stage = 'read-account' + check( + await page.$eval('#account', (node) => node.textContent === 'fixture-account'), + 'Target lost login state' + ) + const response = page.waitForResponse((reply) => reply.url().endsWith('/data?page=2')) + stage = 'click-more' + await page.click('#more') + stage = 'response-json' + const body = await (await response).json() + stage = 'wait-later-batch' + await page.waitForFunction(() => document.querySelectorAll('li').length === 4, { + polling: 100, + }) + return { + body, + entries: await page.$$eval('li', (nodes) => nodes.map((node) => node.textContent)), + } + } + ) + check( + extracted.entries.join(',') === 'one,two,three,four', + 'Page interactions did not load the later batch' + ) + check(extracted.body.entries.length === 2, 'Network response body did not cross the bridge') + check( + (await BrowserAccess.tabs(binding.provider)).length === baseline.length, + 'Owned tab was not closed' + ) + check( + await stalePage!.evaluate('1').then( + () => false, + () => true + ), + 'Old Page still executed after release' + ) + + const borrowed = (await BrowserAccess.tabs(binding.provider)).find((tab) => + tab.url?.endsWith('/borrowed') + ) + check(borrowed, 'Fixture borrowed tab is missing') + stage = 'borrowed-tab' + await BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'existing', tab: borrowed } }, + async (page) => { + await page.evaluate(() => { + document.title = 'borrowed-preserved' + }) + const busyTab = (await BrowserAccess.tabs(binding.provider)).find( + (tab) => tab.tabId === borrowed.tabId + ) + check(busyTab, 'Active borrowed tab disappeared') + check( + await BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'existing', tab: busyTab } }, + async () => null + ).then( + () => false, + () => true + ), + 'Two sessions acquired the same borrowed tab' + ) + } + ) + check( + (await BrowserAccess.tabs(binding.provider)).some((tab) => tab.tabId === borrowed.tabId), + 'Borrowed tab was closed' + ) + check( + await BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'existing', tab: borrowed } }, + async () => null + ).then( + () => false, + () => true + ), + 'Consumed tab reference was reused' + ) + + const controller = new AbortController() + stage = 'abort' + let entered = false + const cancelled = BrowserAccess.withPage( + { provider: binding.provider, target: { kind: 'new' }, signal: controller.signal }, + async (page) => { + await page.goto(`${location.origin}/target`) + entered = true + setTimeout(() => controller.abort(), 50) + await page.waitForSelector('#never', { timeout: 0 }) + } + ) + check( + await cancelled.then( + () => false, + () => true + ), + 'Aborted callback completed successfully' + ) + check(entered, 'Abort did not exercise an active callback') + check( + (await BrowserAccess.tabs(binding.provider)).length === baseline.length, + 'Aborted owned tab was not cleaned up' + ) + + stage = 'provider-revoke-isolation' + const secondBinding = found.find((item) => item.provider === config.secondProvider) + check(secondBinding, 'Second provider is missing') + BrowserAccess.bind(secondBinding) + await BrowserAccess.withPage( + { provider: secondBinding.provider, target: { kind: 'new' } }, + async (page) => { + await page.goto(`${location.origin}/target`) + const revoked = await fetch('/fixture-command', { + method: 'POST', + body: JSON.stringify({ + type: 'browser-automation-revoke', + origin: location.origin, + provider: binding.provider, + }), + }).then((response) => response.json()) + check(revoked.ok, 'Native provider revocation failed') + check( + await page.$$eval('li', (nodes) => nodes.length === 2), + 'Revoking provider A disconnected provider B' + ) + } + ) + const wrongProvider = crypto.randomUUID() + BrowserAccess.bind({ ...binding, provider: wrongProvider }) + check( + await BrowserAccess.tabs(wrongProvider, AbortSignal.timeout(100)).then( + () => false, + () => true + ), + 'Another provider silently substituted for the exact target' + ) + BrowserAccess.unbind(wrongProvider) + const probe = async (route: string): Promise<{ count: number; providers: string[] }> => { + const token = crypto.randomUUID() + const opened = await fetch('/fixture-command', { + method: 'POST', + body: JSON.stringify({ + type: 'fixture-open-probe', + url: `${config.otherOrigin}${route}?token=${token}`, + }), + }).then((response) => response.json()) + check(opened.ok, 'Could not open a separate top-level Web origin') + const observed = await fetch(`/probe-result?token=${token}`).then((response) => response.json()) + await fetch('/fixture-command', { + method: 'POST', + body: JSON.stringify({ type: 'fixture-close-probe', tabId: opened.value.id }), + }) + return observed + } + check( + (await probe('/unpaired')).count === 0, + 'An unpaired origin on the same permitted host acquired a provider' + ) + const granted = await fetch('/fixture-command', { + method: 'POST', + body: JSON.stringify({ + type: 'browser-automation-grant', + origin: config.otherOrigin, + provider: secondBinding.provider, + }), + }).then((response) => response.json()) + check(granted.ok, 'Second origin could not be paired dynamically') + const paired = await probe('/paired') + check( + paired.count === 1 && paired.providers[0] === secondBinding.provider, + 'Second origin did not discover its exact authorized provider' + ) + await fetch('/report', { + method: 'POST', + body: JSON.stringify({ + ok: true, + entries: extracted.entries, + cases: [ + 'provider-revoke-isolation', + 'second-origin-pairing', + 'login-state', + 'page-navigation', + 'dynamic-batch', + 'network-json', + 'new-tab-cleanup', + 'borrowed-tab-preserved', + 'borrowed-tab-exclusion', + 'stale-page', + 'stale-tab-reference', + 'abort-cleanup', + 'exact-provider', + 'exact-origin', + ], + }), + }) + document.querySelector('#status')!.textContent = 'Passed' +} + +void main().catch(async (error: unknown) => { + document.querySelector('#status')!.textContent = String(error) + await fetch('/report', { + method: 'POST', + body: JSON.stringify({ + ok: false, + error: `${stage}: ${error instanceof Error ? `${error.name}: ${error.message}\n${error.stack}` : String(error)}`, + }), + }) +}) diff --git a/scripts/tests/job-cancellation.test.mjs b/scripts/tests/job-cancellation.test.mjs new file mode 100644 index 00000000..3e1640a1 --- /dev/null +++ b/scripts/tests/job-cancellation.test.mjs @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { Job, JobManager, JobStatus, MetaConfigSchema } from '../../packages/core/dist/index.js' + +function pending(id, type, abortRequested) { + return Job.parse({ + id, + type, + parameters: {}, + state: {}, + abort_requested: abortRequested, + timeout_seconds: 30, + status: JobStatus.PENDING, + created_at: new Date(), + started_at: null, + closed_at: null, + }) +} + +function database(t, job, abortWhenClaimed) { + const updates = [] + t.mock.method(Job, 'get', async () => job) + t.mock.method(Job.dbApi, 'update', (fields) => ({ + eq() { + return this + }, + select() { + if (fields.status === JobStatus.RUNNING) + return { + maybeSingle: async () => ({ + data: { ...job, status: JobStatus.RUNNING, abort_requested: abortWhenClaimed }, + }), + } + updates.push(fields.status) + return Promise.resolve({ data: [{ id: job.id }] }) + }, + })) + return updates +} + +void test('an already cancelled pending Job closes without asking business eligibility or running', async (t) => { + const type = `test.cancel.${crypto.randomUUID()}` + const job = pending(1, type, true) + const updates = database(t, job, true) + const canHandle = t.mock.fn(() => false) + const handle = t.mock.fn(async () => undefined) + JobManager.registerHandler(type, { parameters: MetaConfigSchema, canHandle, handle }) + try { + assert.equal(await JobManager.run(job.id), true) + assert.equal(canHandle.mock.callCount(), 0) + assert.equal(handle.mock.callCount(), 0) + assert.deepEqual(updates, [JobStatus.ABORTED]) + } finally { + JobManager.unregisterHandler(type) + } +}) + +void test('a cancellation arriving during atomic claim prevents the handler from starting', async (t) => { + const type = `test.cancel.${crypto.randomUUID()}` + const job = pending(2, type, false) + const updates = database(t, job, true) + const handle = t.mock.fn(async () => undefined) + JobManager.registerHandler(type, { parameters: MetaConfigSchema, canHandle: () => true, handle }) + try { + assert.equal(await JobManager.run(job.id), true) + assert.equal(handle.mock.callCount(), 0) + assert.deepEqual(updates, [JobStatus.ABORTED]) + } finally { + JobManager.unregisterHandler(type) + } +}) + +void test('ordinary ineligible work remains pending', async (t) => { + const type = `test.cancel.${crypto.randomUUID()}` + const job = pending(3, type, false) + const updates = database(t, job, false) + const handle = t.mock.fn(async () => undefined) + JobManager.registerHandler(type, { parameters: MetaConfigSchema, canHandle: () => false, handle }) + try { + assert.equal(await JobManager.run(job.id), false) + assert.equal(handle.mock.callCount(), 0) + assert.deepEqual(updates, []) + } finally { + JobManager.unregisterHandler(type) + } +}) diff --git a/scripts/tests/source-anchor.test.mjs b/scripts/tests/source-anchor.test.mjs new file mode 100644 index 00000000..02b50781 --- /dev/null +++ b/scripts/tests/source-anchor.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { Block, SourceResolver, ResolverContentError } from '../../packages/core/dist/index.js' + +const anchor = (content) => + Block.parse({ + id: 41, + resolver: 'core.source.v1', + storage: null, + content: JSON.stringify(content), + }) + +void test('Source anchors read canonical identity without needing the collector or its config', async () => { + const resolver = new SourceResolver( + anchor({ id: 9, type: 'extensions.bilibili.favorites.v1', nickname: '我的收藏' }) + ) + assert.equal( + await resolver.getText(), + 'Source 9 named 我的收藏 uses extensions.bilibili.favorites.v1.' + ) + assert.equal(await resolver.getSolvedContent(), await resolver.getText()) + assert.equal( + await new SourceResolver(anchor({ id: 10, type: 'source.example' })).getText(), + 'Source 10 uses source.example.' + ) +}) + +void test('Source anchor decoding rejects a parallel config or malformed identity', async () => { + for (const content of [ + { id: '9', type: 'source.example' }, + { id: 9, type: 'source.example', config: { cookie: 'must-not-be-an-anchor' } }, + { id: 9 }, + ]) { + await assert.rejects(new SourceResolver(anchor(content)).getText(), ResolverContentError) + } +}) diff --git a/scripts/verify-browser-automation.mjs b/scripts/verify-browser-automation.mjs new file mode 100644 index 00000000..531d3955 --- /dev/null +++ b/scripts/verify-browser-automation.mjs @@ -0,0 +1,225 @@ +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { createServer } from 'node:http' +import { createRequire } from 'node:module' +import { once } from 'node:events' +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { join, extname } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { chromium } from '@playwright/test' + +const root = fileURLToPath(new URL('../', import.meta.url)) +const require = createRequire(new URL('../apps/web/package.json', import.meta.url)) +const { build } = await import(pathToFileURL(require.resolve('vite')).href) +await mkdir(join(root, '.runtime'), { recursive: true }) +const temporary = await mkdtemp(join(root, '.runtime/browser-automation-')) +const dist = join(temporary, 'web') +const extension = join(temporary, 'extension') +const provider = randomUUID() +const secondProvider = randomUUID() +const nativeCommands = [] +const nativeReplies = new Map() +const probeResults = new Map() +const probeWaiters = new Map() +let appOrigin +let otherOrigin +let webReport +let nativeReport +let done +const finished = new Promise((resolve) => { + done = resolve +}) + +async function handle(request, response) { + const url = new URL(request.url, appOrigin ?? 'http://127.0.0.1') + if (url.pathname === '/config') + return response.end(JSON.stringify({ provider, secondProvider, otherOrigin })) + if (url.pathname === '/state') return response.end(JSON.stringify({ complete: !!webReport })) + if (url.pathname === '/fixture-next') + return response.end(JSON.stringify(nativeCommands.shift() ?? null)) + if (url.pathname === '/probe-result') { + const token = url.searchParams.get('token') + if (probeResults.has(token)) return response.end(JSON.stringify(probeResults.get(token))) + probeWaiters.set(token, (result) => response.end(JSON.stringify(result))) + return + } + if (url.pathname === '/probe-report') { + let body = '' + for await (const chunk of request) body += chunk + const result = JSON.parse(body) + probeResults.set(result.token, result) + probeWaiters.get(result.token)?.(result) + probeWaiters.delete(result.token) + return response.end('ok') + } + if (url.pathname === '/fixture-command') { + let body = '' + for await (const chunk of request) body += chunk + const id = randomUUID() + nativeCommands.push({ id, message: JSON.parse(body) }) + nativeReplies.set(id, (reply) => response.end(JSON.stringify(reply))) + return + } + if (url.pathname === '/fixture-reply') { + let body = '' + for await (const chunk of request) body += chunk + const { id, reply } = JSON.parse(body) + nativeReplies.get(id)?.(reply) + nativeReplies.delete(id) + return response.end('ok') + } + if (url.pathname === '/report' || url.pathname === '/native-report') { + let body = '' + for await (const chunk of request) body += chunk + if (url.pathname === '/report') webReport = JSON.parse(body) + else nativeReport = JSON.parse(body) + response.end('ok') + if (webReport && nativeReport) done() + return + } + if (url.pathname === '/borrowed') + return response.end('borrowed

Borrowed tab

') + if (url.pathname === '/target') { + response.setHeader('Content-Type', 'text/html') + return response.end( + `

${request.headers.cookie?.includes('fixture-login=1') ? 'fixture-account' : 'not-logged-in'}

  • one
  • two
` + ) + } + if (url.pathname === '/data') { + response.setHeader('Content-Type', 'application/json') + return response.end(JSON.stringify({ entries: ['three', 'four'] })) + } + const path = url.pathname.startsWith('/assets/') + ? join(dist, url.pathname) + : join(dist, 'index.html') + try { + response.setHeader( + 'Content-Type', + extname(path) === '.js' + ? 'application/javascript' + : extname(path) === '.css' + ? 'text/css' + : 'text/html' + ) + response.setHeader('Set-Cookie', 'fixture-login=1; SameSite=Lax; Path=/') + response.end(await readFile(path)) + } catch { + response.writeHead(404).end() + } +} +const servers = [ + createServer((request, response) => void handle(request, response)), + createServer((request, response) => void handle(request, response)), +] +for (const server of servers) server.listen(0, '127.0.0.1') +await Promise.all(servers.map((server) => once(server, 'listening'))) +appOrigin = `http://127.0.0.1:${servers[0].address().port}` +otherOrigin = `http://127.0.0.1:${servers[1].address().port}` +let browserProcess +let stderr = '' +let timeout +try { + await build({ + configFile: false, + root: join(root, 'scripts/tests/browser-automation'), + resolve: { alias: { '@inkcre/core': join(root, 'packages/core/src/index.ts') } }, + build: { + outDir: dist, + emptyOutDir: true, + }, + }) + await cp(join(root, 'apps/webext/.output/chrome-mv3'), extension, { recursive: true }) + const manifest = JSON.parse(await readFile(join(extension, 'manifest.json'), 'utf8')) + // This isolated fixture pregrants only its synthetic host. Native permission UX is separate evidence. + manifest.host_permissions = ['http://127.0.0.1/*'] + manifest.background.service_worker = 'automation-fixture-background.js' + manifest.background.type = 'module' + await writeFile(join(extension, 'manifest.json'), JSON.stringify(manifest)) + await writeFile( + join(extension, 'automation-fixture.html'), + '' + ) + await writeFile( + join(extension, 'automation-fixture-background.js'), + "import './background.js'; chrome.runtime.onInstalled.addListener(() => { void chrome.tabs.create({url: chrome.runtime.getURL('automation-fixture.html')}); });" + ) + await writeFile( + join(extension, 'automation-fixture.js'), + ` +(async()=>{ +const origin=${JSON.stringify(appOrigin)},provider=${JSON.stringify(provider)},secondProvider=${JSON.stringify(secondProvider)}; +const db=await new Promise((resolve,reject)=>{const r=indexedDB.open('inkcre-webext-collection',1);r.onupgradeneeded=()=>{for(const name of ['settings','connections','outbox','imports','legacy'])if(!r.result.objectStoreNames.contains(name))r.result.createObjectStore(name)};r.onsuccess=()=>resolve(r.result);r.onerror=()=>reject(r.error)}); +await new Promise((resolve,reject)=>{const tx=db.transaction(['connections'],'readwrite');for(const peerId of [provider,secondProvider]){const id=crypto.randomUUID();tx.objectStore('connections').put({id,name:'Fixture browser',url:origin+'/unused-database',secret:'fixture-only-not-a-real-secret-32-bytes',peerId,storageId:-4},id)}tx.oncomplete=resolve;tx.onerror=()=>reject(tx.error)}); +let result; +for(let i=0;i<30;i++){try{result=await chrome.runtime.sendMessage({type:'browser-automation-grant',origin,provider});if(result?.ok)break}catch{}await new Promise(r=>setTimeout(r,100))} +if(!result?.ok)throw new Error(result?.error??'Background did not initialize'); +const second=await chrome.runtime.sendMessage({type:'browser-automation-grant',origin,provider:secondProvider});if(!second?.ok)throw new Error(second?.error??'Second provider grant failed'); +await chrome.tabs.create({url:origin+'/borrowed',active:false}); +await chrome.tabs.create({url:origin+'/app',active:false}); +for(let i=0;i<1200;i++){const command=await fetch(origin+'/fixture-next').then(r=>r.json());if(command){const reply=command.message.type==='fixture-open-probe'?{ok:true,value:await chrome.tabs.create({url:command.message.url,active:false})}:command.message.type==='fixture-close-probe'?{ok:true,value:await chrome.tabs.remove(command.message.tabId)}:await chrome.runtime.sendMessage(command.message);await fetch(origin+'/fixture-reply',{method:'POST',body:JSON.stringify({id:command.id,reply})})}if((await fetch(origin+'/state').then(r=>r.json())).complete){const targets=await chrome.debugger.getTargets();await fetch(origin+'/native-report',{method:'POST',body:JSON.stringify({attached:targets.filter(t=>t.attached).map(t=>({id:t.id,tabId:t.tabId,url:t.url}))})});return}await new Promise(r=>setTimeout(r,100))} +throw new Error('Web fixture timed out'); +})().catch(async error=>{await fetch(${JSON.stringify(appOrigin)}+'/report',{method:'POST',body:JSON.stringify({ok:false,error:error.stack})});await fetch(${JSON.stringify(appOrigin)}+'/native-report',{method:'POST',body:'{}'})}); +` + ) + const executable = globalThis.process.env.INKCRE_CHROMIUM_BINARY || chromium.executablePath() + browserProcess = spawn( + executable, + [ + '--headless=new', + // Hosted Linux CI disallows Chromium's user-namespace sandbox. This browser + // uses a disposable profile and serves only this fixture's synthetic pages. + ...(process.platform === 'linux' && process.env.CI ? ['--no-sandbox'] : []), + '--no-first-run', + '--no-default-browser-check', + '--disable-background-networking', + `--user-data-dir=${join(temporary, 'profile')}`, + `--disable-extensions-except=${extension}`, + `--load-extension=${extension}`, + '--enable-logging=stderr', + 'about:blank', + ], + { stdio: ['ignore', 'ignore', 'pipe'] } + ) + browserProcess.stderr.on('data', (chunk) => { + stderr = (stderr + chunk).slice(-4000) + }) + browserProcess.on('error', (error) => { + webReport = { ok: false, error: String(error) } + done() + }) + browserProcess.on('exit', (code, signal) => { + if (webReport && nativeReport) return + webReport = { + ok: false, + error: `Browser fixture exited before reporting (code ${code}, signal ${signal})\n${stderr}`, + } + done() + }) + await Promise.race([ + finished, + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(`Browser fixture timed out\n${stderr}`)), 120_000) + }), + ]) + assert.equal(webReport?.ok, true, webReport?.error ?? stderr) + assert.deepEqual( + nativeReport?.attached, + [], + 'An owned debugger attachment survived callback cleanup' + ) + console.log(JSON.stringify({ browser: executable, ...webReport, nativeReport }, null, 2)) +} finally { + clearTimeout(timeout) + if (browserProcess && browserProcess.exitCode === null) { + browserProcess.kill('SIGTERM') + await Promise.race([ + once(browserProcess, 'exit'), + new Promise((resolve) => setTimeout(resolve, 5000)), + ]) + if (browserProcess.exitCode === null) browserProcess.kill('SIGKILL') + } + for (const server of servers) server.close() + // Chromium helpers can still flush the disposable profile after the main process exits. + await rm(temporary, { recursive: true, force: true, maxRetries: 5 }) +} diff --git a/scripts/webext-release.mjs b/scripts/webext-release.mjs index 812be437..2c3a1576 100644 --- a/scripts/webext-release.mjs +++ b/scripts/webext-release.mjs @@ -35,7 +35,8 @@ export function checkManifest(manifest, version, browser, key) { if (browser === 'chrome') assert.equal(manifest.key, key, 'Release must use the fixed Chromium identity') else assert.equal(manifest.browser_specific_settings?.gecko?.id, 'webext@inkcre.org') - assert.ok(!manifest.permissions.includes('debugger')) + assert.equal(manifest.permissions.includes('debugger'), browser === 'chrome') + if (browser === 'chrome') assert.equal(manifest.minimum_chrome_version, '125') } async function api(route, options = {}) { diff --git a/tests/e2e/browser-extension.spec.ts b/tests/e2e/browser-extension.spec.ts index 957abe0e..54399d65 100644 --- a/tests/e2e/browser-extension.spec.ts +++ b/tests/e2e/browser-extension.spec.ts @@ -680,7 +680,7 @@ test('settings preserve the WebExt card style across desktop, narrow and dark la ) await expect(page.getByLabel('PostgREST 地址')).toBeVisible() const layout = await page.evaluate(() => { - const card = document.querySelector('.collection-page > .section')! + const card = document.querySelector('section[aria-labelledby="connection-heading"]')! const input = card.querySelector('input')! const primary = card.querySelector('button[type="submit"]')! const toggle = document.querySelector('[role="switch"]')!