diff --git a/.github/workflows/catalog-listing.yml b/.github/workflows/catalog-listing.yml new file mode 100644 index 0000000..1c86c3b --- /dev/null +++ b/.github/workflows/catalog-listing.yml @@ -0,0 +1,59 @@ +name: Publish Vision catalog listing + +# The catalog listing (catalog/listing.yaml + catalog/screenshots) is +# presentation only: a change never implies a release. On main it replaces the +# listing of the entry the catalog already records and requests a publication. + +on: + push: + branches: [main] + paths: + - catalog/listing.yaml + - catalog/screenshots/** + - hack/catalog-listing.ts + - .github/workflows/catalog-listing.yml + workflow_dispatch: + +permissions: + contents: read + +# Shares the writer with the release publisher: one read/merge/PUT at a time. +concurrency: + group: vision-catalog-publish + cancel-in-progress: false + +jobs: + listing: + if: github.repository == 'appthrust/vision' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 + with: + bun-version: 1.4.1 + - name: Validate the listing + run: bun hack/publish-entry.ts validate-listing + - name: Submit the listing + id: catalog + env: + CLOUD_API_URL: ${{ vars.CLOUD_API_URL || 'https://api.appthrust.com' }} + CLOUD_CATALOG_ADMIN_TOKEN: ${{ secrets.CLOUD_CATALOG_ADMIN_TOKEN }} + run: bun hack/publish-entry.ts listing + - name: Mint the Cloud workflow dispatch token + if: steps.catalog.outputs.state == 'snapshot' + id: cloud-token + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2 + with: + app-id: ${{ secrets.APPTHRUST_GITHUB_ACTIONS_APP_ID }} + private-key: ${{ secrets.APPTHRUST_GITHUB_ACTIONS_APP_PRIVATE_KEY }} + owner: appthrust + repositories: cloud + - name: Dispatch Cloud catalog publication + if: steps.catalog.outputs.state == 'snapshot' + env: + GH_TOKEN: ${{ steps.cloud-token.outputs.token }} + SNAPSHOT_ID: ${{ steps.catalog.outputs.snapshot_id }} + run: gh api -X POST repos/appthrust/cloud/dispatches -f event_type=catalog-publish -f "client_payload[snapshot_id]=$SNAPSHOT_ID" diff --git a/catalog/listing.yaml b/catalog/listing.yaml new file mode 100644 index 0000000..87ed32c --- /dev/null +++ b/catalog/listing.yaml @@ -0,0 +1,64 @@ +locales: + en: + title: Vision + tagline: Read the text in images and describe what they show, with a vision model that runs inside the Project. + overview: | + Vision gives the apps in a Project an image-understanding model. An app sends an image with an instruction and gets text back: the words printed on a business card or document, or a description of what the picture shows. It suits teams whose apps need OCR or image understanding and who want the images to stay inside the Project. + + Apps call Vision in the OpenAI chat completions format: one message holding an image and the instruction, answered in one piece or streamed. An app that already talks to a chat model can add image input without a new client library. + + Contacts can use Vision to read business cards. When the Project's AI gateway offers the Vision model, Contacts lists it as an in-cluster model in its reading settings, and card images are read without leaving the cluster. + + The model is part of the app and runs offline: nothing is downloaded at runtime, and images and answers stay in the Project. Inference needs an NVIDIA GPU; each running copy uses one, so production plus staging uses two. Each request carries one image of up to 8 MiB, and up to four requests run at the same time. Vision has no screen of its own; other apps in the Project send it requests. + ja: + title: Vision + tagline: 画像の文字を読み取り、写っている内容を説明するモデルを Project の中で使えます。 + overview: | + Vision は、Project 内のアプリが使える画像理解モデルです。アプリが画像と指示を送ると、名刺や書類に印刷された文字や、写真に写っている内容の説明を文章で返します。OCR や画像の理解が必要なアプリを動かしていて、画像を Project の外に出したくないチームに向いています。 + + アプリからは OpenAI のチャット補完と同じ形式で呼び出します。画像と指示を 1 通のメッセージにまとめて送り、結果はまとめて受け取ることも、少しずつ受け取ることもできます。すでにチャットモデルを使っているアプリなら、新しいクライアントを用意せずに画像の入力を加えられます。 + + Contacts の名刺読み取りにも使えます。Project の AI ゲートウェイが Vision のモデルを提供していると、Contacts の読み取り設定にクラスタ内のモデルとして表示され、名刺画像はクラスタの外に出ないまま読み取られます。 + + モデルはアプリに含まれていて、オフラインで動きます。実行中に外部から何かを取得することはなく、画像も結果も Project の中にとどまります。推論には NVIDIA GPU が必要です。動いているコピーごとに 1 基使うため、本番とステージングを動かすと 2 基になります。1 回のリクエストで送れる画像は 1 枚、8 MiB までで、同時に処理するのは 4 件までです。Vision 自体に画面はなく、Project 内のほかのアプリから呼び出して使います。 +highlights: + - screenshot: install + locales: + en: + title: Runs on the Project's GPU + body: Each running copy uses one NVIDIA GPU. The install dialog shows the count for each branch before anything starts, so you know that production plus staging takes two. + ja: + title: Project の GPU で動きます + body: 動いているコピーごとに NVIDIA GPU を 1 基使います。インストールの画面で、始める前にブランチごとの数が分かります。本番とステージングなら合計 2 基です。 + - screenshot: catalog + locales: + en: + title: Send an image, get text back + body: Apps send one image and an instruction in the OpenAI chat completions format and receive the answer as text, in one piece or streamed. Vision sits in the Assist catalog next to the Project's text chat models. + ja: + title: 画像を送ると文章で返します + body: アプリは OpenAI のチャット補完と同じ形式で画像 1 枚と指示を送り、答えを文章で受け取ります。まとめて受け取ることも、少しずつ受け取ることもできます。カタログでは文章向けのチャットモデルと同じ Assist に並んでいます。 + - screenshot: versions + locales: + en: + title: The model ships with the app + body: Every version includes the model weights and runs offline, so nothing is fetched at runtime and images stay in the Project. Pin a version, or follow its release line to get updates automatically. + ja: + title: モデルはアプリに同梱 + body: どのバージョンにもモデルの重みが入っていて、オフラインで動きます。実行中に外部から取得するものはなく、画像は Project の外に出ません。バージョンは固定することも、リリースラインに沿って自動で更新することもできます。 +screenshots: + - name: install + file: install.webp + alt: + en: The Dashboard install dialog for Vision with version 0.2.0 selected, update and branch options, and an accelerator table showing one NVIDIA GPU for production, one for staging, two in total. + ja: Dashboard の Vision インストール画面。バージョン 0.2.0、更新とブランチの選択肢、アクセラレータの表(本番 1、ステージング 1、合計 2 NVIDIA GPU)が表示されています。 + - name: catalog + file: catalog.webp + alt: + en: The Marketplace catalog in the Dashboard, showing the Assist apps AI Gateway, Embedding, Inference, Knowledge, LLM and Vision as cards, with an Install button on the Vision card. + ja: Dashboard のマーケットプレイスのカタログ。Assist の AI Gateway、Embedding、Inference、Knowledge、LLM、Vision がカードで並び、Vision のカードにインストールボタンがあります。 + - name: versions + file: versions.webp + alt: + en: The Vision page in the Dashboard Marketplace with its description, an Install button, update options, and the version list 0.1.0 and 0.2.0, with 0.2.0 marked as the default. + ja: Dashboard マーケットプレイスの Vision のページ。説明文、インストールボタン、更新の選択肢、バージョン一覧(0.1.0 と既定の 0.2.0)が表示されています。 diff --git a/catalog/screenshots/catalog.webp b/catalog/screenshots/catalog.webp new file mode 100644 index 0000000..972d918 Binary files /dev/null and b/catalog/screenshots/catalog.webp differ diff --git a/catalog/screenshots/install.webp b/catalog/screenshots/install.webp new file mode 100644 index 0000000..13cbe7b Binary files /dev/null and b/catalog/screenshots/install.webp differ diff --git a/catalog/screenshots/versions.webp b/catalog/screenshots/versions.webp new file mode 100644 index 0000000..1905af6 Binary files /dev/null and b/catalog/screenshots/versions.webp differ diff --git a/go.mod b/go.mod index 9c4b94f..238217b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/appthrust/vision go 1.26.0 require ( - github.com/appthrust/appserver-go v0.1.0 + github.com/appthrust/appserver-go v0.3.0 github.com/stretchr/testify v1.11.1 ) diff --git a/go.sum b/go.sum index 12ad891..d059bbb 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/appthrust/appserver-go v0.1.0 h1:MYpkwwm4wqa+YcKe1AwywcRe9xuUtTooK4+o7Z5Jbqo= -github.com/appthrust/appserver-go v0.1.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= +github.com/appthrust/appserver-go v0.3.0 h1:zxOz0hkspZSXjLq3KWGDLkZIz42Jag6wqRujZ8lTru0= +github.com/appthrust/appserver-go v0.3.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= diff --git a/hack/catalog-listing.test.ts b/hack/catalog-listing.test.ts new file mode 100644 index 0000000..7cb5730 --- /dev/null +++ b/hack/catalog-listing.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, test } from 'bun:test' +import { createHash } from 'node:crypto' +import { mkdtemp, mkdir, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { imageDimensions, loadListing, uploadMedia, validateListing, type Listing } from './catalog-listing' +import { renderEntry, renderListingUpdate, type CatalogEntry } from './publish-entry' + +/** Minimal valid PNG: 8-byte signature + IHDR chunk; dimensions only, no pixel data. */ +function png(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(33) + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0) + const view = new DataView(bytes.buffer) + view.setUint32(8, 13) + bytes.set([0x49, 0x48, 0x44, 0x52], 12) + view.setUint32(16, width) + view.setUint32(20, height) + return bytes +} + +function webpVP8X(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(30) + bytes.set([0x52, 0x49, 0x46, 0x46], 0) + bytes.set([0x57, 0x45, 0x42, 0x50], 8) + bytes.set([0x56, 0x50, 0x38, 0x58], 12) + const w = width - 1, h = height - 1 + bytes.set([w & 0xff, (w >> 8) & 0xff, (w >> 16) & 0xff], 24) + bytes.set([h & 0xff, (h >> 8) & 0xff, (h >> 16) & 0xff], 27) + return bytes +} + +const listing: Listing = { + locales: { + en: { tagline: 'People, groups and profiles.', overview: 'Who is who.\n\nSearch and browse.' }, + ja: { tagline: 'この組織のメンバー、グループ、プロフィール。', overview: '誰が誰か。' }, + }, + highlights: [{ locales: { en: { title: 'Search' }, ja: { title: '検索' } }, screenshot: 'people' }], + screenshots: [{ + name: 'people', digest: `sha256:${'a'.repeat(64)}`, mediaType: 'image/png', size: 33, width: 1280, height: 800, + alt: { en: 'People list', ja: 'メンバー一覧' }, + }], +} + +describe('imageDimensions', () => { + test('reads PNG IHDR and WebP VP8X', () => { + expect(imageDimensions(png(1280, 800))).toEqual({ mediaType: 'image/png', width: 1280, height: 800 }) + expect(imageDimensions(webpVP8X(1440, 900))).toEqual({ mediaType: 'image/webp', width: 1440, height: 900 }) + expect(() => imageDimensions(new Uint8Array(40))).toThrow('PNG or WebP') + }) +}) + +describe('validateListing', () => { + test('accepts the sample and enforces the contract', () => { + expect(() => validateListing(listing)).not.toThrow() + expect(() => validateListing({ locales: { ja: { tagline: 'x' } } })).toThrow('requires the "en" locale') + expect(() => validateListing({ ...listing, highlights: [{ locales: { en: { title: 'x' } }, screenshot: 'missing' }] })) + .toThrow('unknown screenshot "missing"') + expect(() => validateListing({ ...listing, screenshots: [{ ...listing.screenshots![0], size: 600_000 }] })).toThrow('bytes') + expect(() => validateListing({ ...listing, screenshots: [{ ...listing.screenshots![0], alt: { ja: 'x' } }] })).toThrow('requires the "en" locale') + expect(() => validateListing({ locales: { en: { tagline: 'x'.repeat(201) } } })).toThrow('exceeds 200') + }) +}) + +describe('loadListing', () => { + test('resolves screenshot files to digests, sizes and dimensions', async () => { + const dir = await mkdtemp(join(tmpdir(), 'listing-')) + await mkdir(join(dir, 'screenshots')) + const image = png(1200, 750) + await writeFile(join(dir, 'screenshots', 'people.png'), image) + await writeFile(join(dir, 'listing.yaml'), [ + 'locales:', ' en: { tagline: People, groups and profiles. }', ' ja: { tagline: メンバー一覧 }', + 'highlights:', ' - screenshot: people', ' locales: { en: { title: Search }, ja: { title: 検索 } }', + 'screenshots:', ' - name: people', ' file: people.png', ' alt: { en: People list, ja: メンバー一覧 }', + ].join('\n')) + const { listing: loaded, media } = await loadListing(pathToFileURL(`${dir}/`)) + const digest = `sha256:${createHash('sha256').update(image).digest('hex')}` + expect(media).toHaveLength(1) + expect(media[0].bytes).toEqual(image) + expect(loaded.screenshots).toEqual([{ name: 'people', digest, mediaType: 'image/png', size: 33, width: 1200, height: 750, alt: { en: 'People list', ja: 'メンバー一覧' } }]) + expect(loaded.highlights?.[0].screenshot).toBe('people') + }) + + test('rejects paths escaping catalog/screenshots', async () => { + const dir = await mkdtemp(join(tmpdir(), 'listing-')) + await writeFile(join(dir, 'listing.yaml'), 'locales: { en: { tagline: x } }\nscreenshots: [{ name: a, file: ../x.png, alt: { en: a } }]\n') + await expect(loadListing(pathToFileURL(`${dir}/`))).rejects.toThrow('relative path') + }) +}) + +describe('uploadMedia', () => { + test('PUTs each blob by hex digest and checks the stored descriptor', async () => { + const image = png(10, 10) + const digest = `sha256:${createHash('sha256').update(image).digest('hex')}` + const calls: Array<{ url: string; type: string }> = [] + const fetcher = async (url: string, init: RequestInit): Promise => { + calls.push({ url, type: (init.headers as Record)['content-type'] }) + return Response.json({ digest, mediaType: 'image/png', size: 33 }) + } + await uploadMedia([{ ...listing.screenshots![0], digest, bytes: image }], 'https://api.example', 'token', fetcher) + expect(calls).toEqual([{ url: `https://api.example/api/v1/catalog/media/${digest.slice(7)}`, type: 'image/png' }]) + const mismatch = async (): Promise => Response.json({ digest, mediaType: 'image/png', size: 1 }) + await expect(uploadMedia([{ ...listing.screenshots![0], digest, bytes: image }], 'https://api.example', 'token', mismatch)) + .rejects.toThrow('does not match') + }) +}) + +describe('listing in entries', () => { + const template = [ + 'apiVersion: appthrust.io/v1alpha1', 'kind: ApplicationStackDefinition', 'metadata: { name: vision }', + 'spec:', ' display: { title: Directory, description: People., category: collaborate, iconRef: { name: vision } }', + ' versions: []', ' updatePolicy: { default: pinned, allowed: [pinned, follow] }', + ].join('\n') + const digest = `sha256:${'b'.repeat(64)}` + + test('renderEntry carries the authored listing and otherwise preserves the stored one', () => { + const first = renderEntry(template, '0.2.0', digest, undefined, listing) + expect(first.listing).toEqual(listing) + const next = renderEntry(template, '0.2.1', `sha256:${'c'.repeat(64)}`, first) + expect(next.listing).toEqual(listing) + expect(renderEntry(template, '0.2.0', digest).listing).toBeUndefined() + }) + + test('renderListingUpdate replaces only the listing', () => { + const current: CatalogEntry = { ...renderEntry(template, '0.2.0', digest), publishedAt: '2026-11-01T00:00:00Z' } + const updated = renderListingUpdate(current, listing) + expect(updated).toEqual({ ...current, listing }) + expect(() => renderListingUpdate({ ...current, slug: 'other' }, listing)).toThrow('slug vision') + }) +}) diff --git a/hack/catalog-listing.ts b/hack/catalog-listing.ts new file mode 100644 index 0000000..12cc073 --- /dev/null +++ b/hack/catalog-listing.ts @@ -0,0 +1,201 @@ +// Catalog listing: the publisher-authored presentation of this app (localized +// copy, highlights, digest-pinned screenshots). Authored in catalog/listing.yaml +// with screenshot files under catalog/screenshots/, uploaded to the Cloud +// catalog media store, and submitted inside the catalog entry. The platform +// projects it onto ApplicationStackDefinition.spec.display and the Dashboard +// renders it on the Marketplace card and detail page. +// +// Bounds mirror the platform contract (pkg/appcatalog.ValidateListing): +// locales ≤ 16 with "en" required; tagline ≤ 200, title ≤ 100, overview ≤ 4000, +// notes ≤ 2000 per version (≤ 32 versions); highlights ≤ 6 (title ≤ 80, +// body ≤ 400); screenshots ≤ 8, each ≤ 512 KiB image/webp or image/png with +// alt ≤ 300 per locale. +import { createHash } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import { YAML } from 'bun' + +export const FALLBACK_LOCALE = 'en' +export const MAX_MEDIA_SIZE = 512 * 1024 + +export interface ListingLocale { + title?: string + tagline: string + overview?: string + notes?: Record +} + +export interface ListingHighlight { + locales: Record + screenshot?: string +} + +export interface ListingMedia { + name: string + digest: string + mediaType: 'image/webp' | 'image/png' + size: number + width: number + height: number + alt: Record +} + +export interface Listing { + locales: Record + highlights?: ListingHighlight[] + screenshots?: ListingMedia[] +} + +/** catalog/listing.yaml: screenshots name a file instead of a digest. */ +interface ListingSource { + locales: Record + highlights?: ListingHighlight[] + screenshots?: Array<{ name: string; file: string; alt: Record }> +} + +export interface MediaBlob extends ListingMedia { + bytes: Uint8Array +} + +const localeTag = /^[a-z]{2,3}$/ +const dnsLabel = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/ +const versionName = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/ + +function bound(value: string | undefined, max: number, what: string): void { + if (value !== undefined && value.length > max) throw new Error(`${what} exceeds ${max} characters`) +} + +function localizedText(text: Record | undefined, max: number, what: string): void { + if (!text || typeof text !== 'object') throw new Error(`${what} must map locales to text`) + if (!(FALLBACK_LOCALE in text)) throw new Error(`${what} requires the "${FALLBACK_LOCALE}" locale`) + if (Object.keys(text).length > 16) throw new Error(`${what} has more than 16 locales`) + for (const [tag, value] of Object.entries(text)) { + if (!localeTag.test(tag)) throw new Error(`${what} locale "${tag}" must be a BCP 47 primary tag`) + if (!value || value.length > max) throw new Error(`${what} locale "${tag}" must be 1-${max} characters`) + } +} + +export function validateListing(listing: Listing): void { + if (!listing.locales || typeof listing.locales !== 'object') throw new Error('listing.locales is required') + if (!(FALLBACK_LOCALE in listing.locales)) throw new Error(`listing requires the "${FALLBACK_LOCALE}" locale`) + if (Object.keys(listing.locales).length > 16) throw new Error('listing has more than 16 locales') + for (const [tag, locale] of Object.entries(listing.locales)) { + if (!localeTag.test(tag)) throw new Error(`listing locale "${tag}" must be a BCP 47 primary tag`) + if (typeof locale.tagline !== 'string' || !locale.tagline) throw new Error(`listing locale "${tag}" requires a tagline`) + bound(locale.title, 100, `locale ${tag} title`) + bound(locale.tagline, 200, `locale ${tag} tagline`) + bound(locale.overview, 4000, `locale ${tag} overview`) + const notes = locale.notes ?? {} + if (Object.keys(notes).length > 32) throw new Error(`locale ${tag} has more than 32 version notes`) + for (const [version, text] of Object.entries(notes)) { + if (!versionName.test(version)) throw new Error(`locale ${tag} notes version "${version}" is invalid`) + bound(text, 2000, `locale ${tag} notes for ${version}`) + } + } + const screenshots = listing.screenshots ?? [] + if (screenshots.length > 8) throw new Error('listing has more than 8 screenshots') + const names = new Set() + for (const media of screenshots) { + if (!dnsLabel.test(media.name) || media.name.length > 63 || names.has(media.name)) { + throw new Error(`screenshot name "${media.name}" is invalid or duplicated`) + } + names.add(media.name) + if (!/^sha256:[a-f0-9]{64}$/.test(media.digest)) throw new Error(`screenshot ${media.name} must pin a sha256 digest`) + if (media.mediaType !== 'image/webp' && media.mediaType !== 'image/png') { + throw new Error(`screenshot ${media.name} has unsupported media type ${media.mediaType}`) + } + if (!(media.size > 0 && media.size <= MAX_MEDIA_SIZE)) throw new Error(`screenshot ${media.name} must be 1-${MAX_MEDIA_SIZE} bytes`) + if (!(media.width > 0 && media.height > 0)) throw new Error(`screenshot ${media.name} has invalid dimensions`) + localizedText(media.alt, 300, `screenshot ${media.name} alt`) + } + const highlights = listing.highlights ?? [] + if (highlights.length > 6) throw new Error('listing has more than 6 highlights') + highlights.forEach((highlight, index) => { + if (!highlight.locales || !(FALLBACK_LOCALE in highlight.locales)) throw new Error(`highlight ${index} requires the "${FALLBACK_LOCALE}" locale`) + if (Object.keys(highlight.locales).length > 16) throw new Error(`highlight ${index} has more than 16 locales`) + for (const [tag, locale] of Object.entries(highlight.locales)) { + if (!localeTag.test(tag) || !locale.title || locale.title.length > 80 || (locale.body ?? '').length > 400) { + throw new Error(`highlight ${index} locale "${tag}" is invalid`) + } + } + if (highlight.screenshot !== undefined && !names.has(highlight.screenshot)) { + throw new Error(`highlight ${index} references unknown screenshot "${highlight.screenshot}"`) + } + }) +} + +/** Image dimensions from the PNG IHDR or WebP VP8/VP8L/VP8X header. */ +export function imageDimensions(bytes: Uint8Array): { mediaType: ListingMedia['mediaType']; width: number; height: number } { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + if (bytes.length >= 24 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) { + return { mediaType: 'image/png', width: view.getUint32(16), height: view.getUint32(20) } + } + if (bytes.length >= 30 && String.fromCharCode(...bytes.subarray(0, 4)) === 'RIFF' && String.fromCharCode(...bytes.subarray(8, 12)) === 'WEBP') { + const chunk = String.fromCharCode(...bytes.subarray(12, 16)) + if (chunk === 'VP8X') { + const width = 1 + (bytes[24] | (bytes[25] << 8) | (bytes[26] << 16)) + const height = 1 + (bytes[27] | (bytes[28] << 8) | (bytes[29] << 16)) + return { mediaType: 'image/webp', width, height } + } + if (chunk === 'VP8L') { + const b0 = bytes[21], b1 = bytes[22], b2 = bytes[23], b3 = bytes[24] + return { mediaType: 'image/webp', width: 1 + (((b1 & 0x3f) << 8) | b0), height: 1 + (((b3 & 0x0f) << 10) | (b2 << 2) | ((b1 & 0xc0) >> 6)) } + } + if (chunk === 'VP8 ') { + return { mediaType: 'image/webp', width: view.getUint16(26, true) & 0x3fff, height: view.getUint16(28, true) & 0x3fff } + } + } + throw new Error('screenshot must be a PNG or WebP image') +} + +/** Reads catalog/listing.yaml and its screenshot files; returns the listing with digests and the blobs to upload. */ +export async function loadListing(catalogDir: URL): Promise<{ listing: Listing; media: MediaBlob[] }> { + const source = YAML.parse(await readFile(new URL('listing.yaml', catalogDir), 'utf8')) as ListingSource + const media: MediaBlob[] = [] + for (const shot of source.screenshots ?? []) { + if (typeof shot.file !== 'string' || shot.file.includes('..') || shot.file.startsWith('/')) { + throw new Error(`screenshot ${shot.name} file must be a relative path under catalog/screenshots`) + } + const bytes = new Uint8Array(await readFile(new URL(`screenshots/${shot.file}`, catalogDir))) + const { mediaType, width, height } = imageDimensions(bytes) + media.push({ + name: shot.name, + digest: `sha256:${createHash('sha256').update(bytes).digest('hex')}`, + mediaType, + size: bytes.byteLength, + width, + height, + alt: shot.alt, + bytes, + }) + } + const listing: Listing = { + locales: source.locales, + ...(source.highlights ? { highlights: source.highlights } : {}), + ...(media.length ? { screenshots: media.map(({ bytes: _bytes, ...descriptor }) => descriptor) } : {}), + } + validateListing(listing) + return { listing, media } +} + +/** Idempotent immutable uploads; the Cloud verifies the body digest against the path. */ +export async function uploadMedia( + media: MediaBlob[], + baseURL: string, + token: string, + fetcher: (url: string, init: RequestInit) => Promise = fetch, +): Promise { + for (const blob of media) { + const response = await fetcher(`${baseURL}/api/v1/catalog/media/${blob.digest.slice(7)}`, { + method: 'PUT', + headers: { authorization: `Bearer ${token}`, 'content-type': blob.mediaType }, + body: blob.bytes as Uint8Array, + signal: AbortSignal.timeout(60_000), + redirect: 'error', + }) + if (response.status !== 200) throw new Error(`PUT /catalog/media/${blob.name} failed (${response.status})`) + const stored = await response.json() as { digest?: string; size?: number; mediaType?: string } + if (stored.digest !== blob.digest || stored.size !== blob.size || stored.mediaType !== blob.mediaType) { + throw new Error(`stored media ${blob.name} does not match the local file`) + } + } +} diff --git a/hack/publish-entry.ts b/hack/publish-entry.ts index de8be75..7833e64 100644 --- a/hack/publish-entry.ts +++ b/hack/publish-entry.ts @@ -3,6 +3,7 @@ import { execFile } from 'node:child_process' import { appendFile } from 'node:fs/promises' import { parseArgs, promisify } from 'node:util' import { YAML } from 'bun' +import { loadListing, type Listing, type MediaBlob, uploadMedia, validateListing } from './catalog-listing' const catalogRepository = 'ghcr.io/appthrust/catalog' const imageRepository = 'ghcr.io/appthrust/vision' @@ -25,6 +26,7 @@ export interface CatalogEntry { versions: CatalogVersion[] updatePolicy: { default: 'pinned' | 'follow'; allowed: Array<'pinned' | 'follow'> } display: { category?: string; icon?: string } + listing?: Listing publishedAt: string | null } @@ -48,7 +50,7 @@ export function validateRelease(version: string, digest: string): void { } } -export function renderEntry(template: string, version: string, digest: string, previous?: CatalogEntry): CatalogEntry { +export function renderEntry(template: string, version: string, digest: string, previous?: CatalogEntry, listing?: Listing): CatalogEntry { validateRelease(version, digest) const definition = YAML.parse(template) as Definition if (definition.apiVersion !== 'appthrust.io/v1alpha1' || definition.kind !== 'ApplicationStackDefinition' || definition.metadata.name !== 'vision') { @@ -81,16 +83,31 @@ export function renderEntry(template: string, version: string, digest: string, p default: true, }) definition.spec.versions = [...versions.values()] + if (listing) validateListing(listing) + const carried = listing ?? previous?.listing return { slug: 'vision', definitionYAML: YAML.stringify(definition), versions: definition.spec.versions, updatePolicy: definition.spec.updatePolicy, display: { category: definition.spec.display.category, icon: definition.spec.display.iconRef?.name }, + ...(carried ? { listing: carried } : {}), publishedAt: previous?.publishedAt ?? null, } } +/** + * Replaces only the listing of the entry the catalog already records. Versions, + * policy and display stay exactly as stored: a listing change never implies a + * release. Requires a published entry; the first tag publish carries the + * listing itself. + */ +export function renderListingUpdate(current: CatalogEntry, listing: Listing): CatalogEntry { + if (current.slug !== 'vision') throw new Error('current entry must have slug vision') + validateListing(listing) + return { ...current, listing } +} + export function catalogContainsRelease(index: unknown, version: string, digest: string): boolean { if (!index || typeof index !== 'object' || !('schemaVersion' in index) || index.schemaVersion !== 1 || !('entries' in index) || !Array.isArray(index.entries)) return false @@ -132,15 +149,46 @@ export interface CatalogPublication { state: 'queued' | 'snapshot' } -export async function submitEntry( - template: string, +/** + * The image digest the catalog already records for `version`, or null. A + * re-run of the publish workflow (for example after a failed Cloud signing + * dispatch) must reuse this image instead of building a new one: versions are + * immutable, so a second build with a different digest is rejected. + */ +export async function recordedDigest( version: string, - digest: string, baseURL: string, token: string, fetcher: (url: string, init: RequestInit) => Promise = fetch, -): Promise { - validateRelease(version, digest) +): Promise { + const response = await fetcher(`${baseURL}/api/v1/catalog/entries`, { + method: 'GET', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + signal: AbortSignal.timeout(30_000), + redirect: 'error', + }) + if (response.status !== 200) throw new Error(`GET /catalog/entries failed (${response.status})`) + const current = await response.json() as { data: CatalogEntry[] } + if (!Array.isArray(current.data)) throw new Error('catalog entries response must contain data[]') + const recorded = current.data.find(entry => entry.slug === 'vision')?.versions.find(entry => entry.name === version) + if (!recorded) return null + const image = recorded.images.web + if (typeof image !== 'string' || !image.startsWith(`${imageRepository}@`) || Object.keys(recorded.images).length !== 1) { + throw new Error(`version ${version} is recorded with an unexpected image set`) + } + const digest = image.slice(imageRepository.length + 1) + if (!digestPattern.test(digest)) throw new Error(`version ${version} is recorded with an invalid digest`) + return digest +} + +interface CatalogClient { + request(path: string, method: string, body?: unknown): Promise + entryRequest(path: string, method: string, body?: unknown): Promise + currentEntry(): Promise + publish(): Promise +} + +function catalogClient(baseURL: string, token: string, fetcher: (url: string, init: RequestInit) => Promise): CatalogClient { async function request(path: string, method: string, body?: unknown): Promise { return fetcher(`${baseURL}/api/v1/catalog/${path}`, { method, @@ -155,25 +203,102 @@ export async function submitEntry( if (response.status !== 200) throw new Error(`${method} /catalog/${path} failed (${response.status})`) return response.json() } - const current = await entryRequest('entries', 'GET') as { data: CatalogEntry[] } - if (!Array.isArray(current.data)) throw new Error('catalog entries response must contain data[]') - const entry = renderEntry(template, version, digest, current.data.find(entry => entry.slug === 'vision')) - await entryRequest('entries/vision', 'PUT', entry) - const response = await request('publish', 'POST') - if (response.status !== 200 && response.status !== 202) throw new Error(`POST /catalog/publish failed (${response.status})`) - const publication = await response.json() as CatalogPublication - if (typeof publication.id !== 'string' || !publication.id || /[\r\n]/.test(publication.id) || - !(response.status === 202 && publication.state === 'queued' || response.status === 200 && publication.state === 'snapshot')) { - throw new Error('catalog publish response must contain a queued (202) or snapshot (200) snapshot id') + return { + request, + entryRequest, + async currentEntry() { + const current = await entryRequest('entries', 'GET') as { data: CatalogEntry[] } + if (!Array.isArray(current.data)) throw new Error('catalog entries response must contain data[]') + return current.data.find(entry => entry.slug === 'vision') + }, + async publish() { + const response = await request('publish', 'POST') + if (response.status !== 200 && response.status !== 202) throw new Error(`POST /catalog/publish failed (${response.status})`) + const publication = await response.json() as CatalogPublication + if (typeof publication.id !== 'string' || !publication.id || /[\r\n]/.test(publication.id) || + !(response.status === 202 && publication.state === 'queued' || response.status === 200 && publication.state === 'snapshot')) { + throw new Error('catalog publish response must contain a queued (202) or snapshot (200) snapshot id') + } + return publication + }, } - return publication +} + +export async function submitEntry( + template: string, + version: string, + digest: string, + baseURL: string, + token: string, + fetcher: (url: string, init: RequestInit) => Promise = fetch, + listing?: { listing: Listing; media: MediaBlob[] }, +): Promise { + validateRelease(version, digest) + const client = catalogClient(baseURL, token, fetcher) + const previous = await client.currentEntry() + if (listing) await uploadMedia(listing.media, baseURL, token, fetcher) + const entry = renderEntry(template, version, digest, previous, listing?.listing) + await client.entryRequest('entries/vision', 'PUT', entry) + return client.publish() +} + +/** Uploads screenshots, replaces the stored entry's listing, and requests a publication. */ +export async function submitListing( + source: { listing: Listing; media: MediaBlob[] }, + baseURL: string, + token: string, + fetcher: (url: string, init: RequestInit) => Promise = fetch, +): Promise { + const client = catalogClient(baseURL, token, fetcher) + const current = await client.currentEntry() + if (!current) throw new Error('catalog has no vision entry yet; publish a release tag first') + await uploadMedia(source.media, baseURL, token, fetcher) + await client.entryRequest('entries/vision', 'PUT', renderListingUpdate(current, source.listing)) + return client.publish() +} + +async function optionalListing(): Promise<{ listing: Listing; media: MediaBlob[] } | undefined> { + const catalogDir = new URL('../catalog/', import.meta.url) + if (!(await Bun.file(new URL('listing.yaml', catalogDir)).exists())) return undefined + return loadListing(catalogDir) } async function main(): Promise { const [subcommand, ...args] = Bun.argv.slice(2) const { values } = parseArgs({ args, options: { version: { type: 'string' }, digest: { type: 'string' } }, strict: true }) + if (subcommand === 'recorded') { + if (!values.version || values.digest) throw new Error('usage: bun hack/publish-entry.ts recorded --version ') + validateRelease(values.version, `sha256:${'1'.repeat(64)}`) + const token = process.env.CLOUD_CATALOG_ADMIN_TOKEN + if (!token) throw new Error('CLOUD_CATALOG_ADMIN_TOKEN is required') + const baseURL = (process.env.CLOUD_API_URL ?? 'https://api.appthrust.com').replace(/\/$/, '') + const digest = await recordedDigest(values.version, baseURL, token) + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `digest=${digest ?? ''}\n`) + console.log(digest ? `Catalog already records ${values.version} as ${digest}` : `Catalog has no image for ${values.version}`) + return + } + if (subcommand === 'validate-listing') { + const source = await optionalListing() + if (!source) throw new Error('catalog/listing.yaml not found') + console.log(`Listing valid: ${Object.keys(source.listing.locales).join(',')} locales, ${source.listing.highlights?.length ?? 0} highlights, ${source.media.length} screenshots (${source.media.reduce((sum, media) => sum + media.size, 0)} bytes)`) + return + } + if (subcommand === 'listing') { + if (values.version || values.digest) throw new Error('usage: bun hack/publish-entry.ts listing') + const token = process.env.CLOUD_CATALOG_ADMIN_TOKEN + if (!token) throw new Error('CLOUD_CATALOG_ADMIN_TOKEN is required') + const baseURL = (process.env.CLOUD_API_URL ?? 'https://api.appthrust.com').replace(/\/$/, '') + const source = await optionalListing() + if (!source) throw new Error('catalog/listing.yaml not found') + const publication = await submitListing(source, baseURL, token) + if (process.env.GITHUB_OUTPUT) { + await appendFile(process.env.GITHUB_OUTPUT, `snapshot_id=${publication.id}\nstate=${publication.state}\n`) + } + console.log(`Catalog snapshot ${publication.id}: ${publication.state}`) + return + } if ((subcommand !== 'submit' && subcommand !== 'await') || !values.version || !values.digest) { - throw new Error('usage: bun hack/publish-entry.ts --version --digest ') + throw new Error('usage: bun hack/publish-entry.ts --version [--digest ]') } validateRelease(values.version, values.digest) if (subcommand === 'submit') { @@ -181,7 +306,7 @@ async function main(): Promise { if (!token) throw new Error('CLOUD_CATALOG_ADMIN_TOKEN is required') const baseURL = (process.env.CLOUD_API_URL ?? 'https://api.appthrust.com').replace(/\/$/, '') const template = await Bun.file(new URL('../catalog/entry.yaml', import.meta.url)).text() - const publication = await submitEntry(template, values.version, values.digest, baseURL, token) + const publication = await submitEntry(template, values.version, values.digest, baseURL, token, fetch, await optionalListing()) if (process.env.GITHUB_OUTPUT) { await appendFile(process.env.GITHUB_OUTPUT, `snapshot_id=${publication.id}\nstate=${publication.state}\n`) }