diff --git a/.github/workflows/catalog-listing.yml b/.github/workflows/catalog-listing.yml new file mode 100644 index 0000000..33afe89 --- /dev/null +++ b/.github/workflows/catalog-listing.yml @@ -0,0 +1,59 @@ +name: Publish LLM catalog listing + +# The catalog listing (catalog/listing.yaml + catalog/screenshots) is +# presentation only: a change never implies a release. On main it replaces the +# listing of the entry the catalog already records and requests a publication. + +on: + push: + branches: [main] + paths: + - catalog/listing.yaml + - catalog/screenshots/** + - hack/catalog-listing.ts + - .github/workflows/catalog-listing.yml + workflow_dispatch: + +permissions: + contents: read + +# Shares the writer with the release publisher: one read/merge/PUT at a time. +concurrency: + group: llm-catalog-publish + cancel-in-progress: false + +jobs: + listing: + if: github.repository == 'appthrust/llm' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 + with: + bun-version: 1.4.1 + - name: Validate the listing + run: bun hack/publish-entry.ts validate-listing + - name: Submit the listing + id: catalog + env: + CLOUD_API_URL: ${{ vars.CLOUD_API_URL || 'https://api.appthrust.com' }} + CLOUD_CATALOG_ADMIN_TOKEN: ${{ secrets.CLOUD_CATALOG_ADMIN_TOKEN }} + run: bun hack/publish-entry.ts listing + - name: Mint the Cloud workflow dispatch token + if: steps.catalog.outputs.state == 'snapshot' + id: cloud-token + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2 + with: + app-id: ${{ secrets.APPTHRUST_GITHUB_ACTIONS_APP_ID }} + private-key: ${{ secrets.APPTHRUST_GITHUB_ACTIONS_APP_PRIVATE_KEY }} + owner: appthrust + repositories: cloud + - name: Dispatch Cloud catalog publication + if: steps.catalog.outputs.state == 'snapshot' + env: + GH_TOKEN: ${{ steps.cloud-token.outputs.token }} + SNAPSHOT_ID: ${{ steps.catalog.outputs.snapshot_id }} + run: gh api -X POST repos/appthrust/cloud/dispatches -f event_type=catalog-publish -f "client_payload[snapshot_id]=$SNAPSHOT_ID" diff --git a/catalog/listing.yaml b/catalog/listing.yaml new file mode 100644 index 0000000..03392d1 --- /dev/null +++ b/catalog/listing.yaml @@ -0,0 +1,64 @@ +locales: + en: + title: LLM + tagline: Use generative AI in your Project's apps without prompts or documents leaving the cluster. + overview: | + LLM runs a chat model on CPU inside your Project, so apps can summarise, answer questions and rewrite text without sending prompts or documents to an outside AI service. It is meant for teams that want AI features but keep their data inside their own environment. + + Apps do not call LLM directly. They send requests to the Project's AI Gateway, which lists LLM as an in-cluster provider next to any external providers and applies the Project's AI policy: which providers and models are allowed and what is recorded. Knowledge, for example, uses the gateway for Ask answers, reranking and query expansion. + + The model (gemma3:4b) is built into each release; nothing is downloaded at startup, and a new model arrives as a new version through the installation's update setting. LLM speaks the OpenAI-compatible chat completions API, including streamed replies, and its model list states the context window (16,384 tokens) and the recommended prompt budget (3,000 tokens) so apps can size their requests. It handles two requests at a time; further requests get an immediate busy response instead of waiting in a queue. + + LLM needs no GPU node, but it reserves several gigabytes of memory in the Project. It reports ready only after the model has answered a short test prompt, and it restarts and warms up again on its own if the model process stops. Prompts, documents and answers stay inside the cluster, and the model runtime's cloud features are turned off. + ja: + title: LLM + tagline: プロンプトや文書をクラスタの外に出さずに、Project 内のアプリで生成 AI を使えます。 + overview: | + LLM は、Project の中でチャットモデルを CPU で動かすアプリです。要約や質問への回答、文章の書き換えを、プロンプトや文書を外部の AI サービスへ送らずに行えます。AI の機能は使いたいが、データは自分たちの環境から出したくないチームに向いています。 + + アプリは LLM を直接呼び出しません。Project の AI ゲートウェイにリクエストを送り、ゲートウェイが LLM をクラスタ内のプロバイダーとして外部のプロバイダーと並べて扱います。どのプロバイダーとモデルを使ってよいか、何を記録するかは、Project の AI ポリシーで決まります。たとえば Knowledge は、質問への回答、検索結果の並べ替え、クエリの補完にゲートウェイを使います。 + + モデル(gemma3:4b)はリリースごとに組み込まれていて、起動時のダウンロードはありません。モデルの更新は新しいバージョンとして届き、インストールの更新設定に沿って反映されます。API は OpenAI 互換のチャット補完で、ストリーミング応答にも対応しています。モデル一覧ではコンテキスト長(16,384 トークン)と推奨のプロンプト上限(3,000 トークン)を返すので、アプリ側でリクエストの大きさを調整できます。同時に処理するのは 2 件までで、それを超えたリクエストは待たせずにすぐ「混雑中」と返します。 + + GPU ノードは不要ですが、Project 内で数 GB のメモリを確保します。短いテスト用の問いかけにモデルが答えてはじめて準備完了になり、モデルのプロセスが止まった場合は自動で再起動して準備し直します。プロンプト、文書、回答はクラスタの外に出ず、モデル実行環境のクラウド機能も無効にしています。 +highlights: + - screenshot: installation + locales: + en: + title: Runs inside your Project + body: The chat model runs as an app in your Project, on CPU. You need no GPU node and no account with an outside AI service, and the installation page shows whether it is ready and healthy. + ja: + title: Project の中で動く + body: チャットモデルは Project 内のアプリとして CPU で動きます。GPU ノードも外部 AI サービスのアカウントも要りません。準備完了かどうか、正常かどうかはインストール画面で確認できます。 + - screenshot: ai-policy + locales: + en: + title: A provider for the AI Gateway + body: The Project's AI policy lists LLM and its model gemma3:4b next to external providers. Apps call the AI Gateway, and the policy decides which providers and models may answer and what is recorded. + ja: + title: AI ゲートウェイのプロバイダーとして + body: Project の AI ポリシーには、LLM とモデル gemma3:4b が外部のプロバイダーと並んで表示されます。アプリは AI ゲートウェイを呼び出し、どのプロバイダーとモデルが答えてよいか、何を記録するかはポリシーで決まります。 + - screenshot: capability + locales: + en: + title: Healthy means it can answer + body: LLM offers OpenAI-compatible chat completions and reports ready only after the model has answered a test prompt. If the model process stops, it restarts and warms up again on its own. + ja: + title: 答えられてはじめて「正常」 + body: LLM は OpenAI 互換のチャット補完を提供し、テスト用の問いかけにモデルが答えてから準備完了になります。モデルのプロセスが止まった場合も、自動で再起動して準備し直します。 +screenshots: + - name: installation + file: installation.webp + alt: + en: "AppThrust Dashboard installation page for LLM version 0.2.0, with Ready and Healthy badges, the app's address, and branch and update settings." + ja: "AppThrust ダッシュボードの LLM(バージョン 0.2.0)のインストール画面。「準備完了」「正常」のバッジ、アプリのアドレス、ブランチと更新の設定が表示されています。" + - name: ai-policy + file: ai-policy.webp + alt: + en: "The Project's effective AI policy in the Dashboard, listing the provider llm (llm.internal) with models gemma3* and gemma3:4b above the openai provider, plus approval modes, logging and 90-day retention." + ja: "ダッシュボードに表示された Project の有効な AI ポリシー。プロバイダー llm(llm.internal)とモデル gemma3*、gemma3:4b が openai の上に並び、承認モード、記録の設定、保持期間 90 日が表示されています。" + - name: capability + file: capability.webp + alt: + en: "Details of the LLM installation: catalog entry llm, application builtin-app-llm, capability chat-completions.openai/v1, and a conditions table where Ready and Healthy are True." + ja: "LLM のインストール詳細。カタログのエントリ llm、アプリケーション builtin-app-llm、機能 chat-completions.openai/v1 と、Ready と Healthy が True になっている条件の一覧が表示されています。" diff --git a/catalog/screenshots/ai-policy.webp b/catalog/screenshots/ai-policy.webp new file mode 100644 index 0000000..280aa4e Binary files /dev/null and b/catalog/screenshots/ai-policy.webp differ diff --git a/catalog/screenshots/capability.webp b/catalog/screenshots/capability.webp new file mode 100644 index 0000000..c463a90 Binary files /dev/null and b/catalog/screenshots/capability.webp differ diff --git a/catalog/screenshots/installation.webp b/catalog/screenshots/installation.webp new file mode 100644 index 0000000..def9223 Binary files /dev/null and b/catalog/screenshots/installation.webp differ diff --git a/go.mod b/go.mod index 8b273dd..8ae2cf1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/appthrust/llm go 1.26.0 require ( - github.com/appthrust/appserver-go v0.1.0 + github.com/appthrust/appserver-go v0.3.0 github.com/stretchr/testify v1.11.1 ) diff --git a/go.sum b/go.sum index 12ad891..d059bbb 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/appthrust/appserver-go v0.1.0 h1:MYpkwwm4wqa+YcKe1AwywcRe9xuUtTooK4+o7Z5Jbqo= -github.com/appthrust/appserver-go v0.1.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= +github.com/appthrust/appserver-go v0.3.0 h1:zxOz0hkspZSXjLq3KWGDLkZIz42Jag6wqRujZ8lTru0= +github.com/appthrust/appserver-go v0.3.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= diff --git a/hack/catalog-listing.test.ts b/hack/catalog-listing.test.ts new file mode 100644 index 0000000..c478224 --- /dev/null +++ b/hack/catalog-listing.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, test } from 'bun:test' +import { createHash } from 'node:crypto' +import { mkdtemp, mkdir, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { imageDimensions, loadListing, uploadMedia, validateListing, type Listing } from './catalog-listing' +import { renderEntry, renderListingUpdate, type CatalogEntry } from './publish-entry' + +/** Minimal valid PNG: 8-byte signature + IHDR chunk; dimensions only, no pixel data. */ +function png(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(33) + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0) + const view = new DataView(bytes.buffer) + view.setUint32(8, 13) + bytes.set([0x49, 0x48, 0x44, 0x52], 12) + view.setUint32(16, width) + view.setUint32(20, height) + return bytes +} + +function webpVP8X(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(30) + bytes.set([0x52, 0x49, 0x46, 0x46], 0) + bytes.set([0x57, 0x45, 0x42, 0x50], 8) + bytes.set([0x56, 0x50, 0x38, 0x58], 12) + const w = width - 1, h = height - 1 + bytes.set([w & 0xff, (w >> 8) & 0xff, (w >> 16) & 0xff], 24) + bytes.set([h & 0xff, (h >> 8) & 0xff, (h >> 16) & 0xff], 27) + return bytes +} + +const listing: Listing = { + locales: { + en: { tagline: 'People, groups and profiles.', overview: 'Who is who.\n\nSearch and browse.' }, + ja: { tagline: 'この組織のメンバー、グループ、プロフィール。', overview: '誰が誰か。' }, + }, + highlights: [{ locales: { en: { title: 'Search' }, ja: { title: '検索' } }, screenshot: 'people' }], + screenshots: [{ + name: 'people', digest: `sha256:${'a'.repeat(64)}`, mediaType: 'image/png', size: 33, width: 1280, height: 800, + alt: { en: 'People list', ja: 'メンバー一覧' }, + }], +} + +describe('imageDimensions', () => { + test('reads PNG IHDR and WebP VP8X', () => { + expect(imageDimensions(png(1280, 800))).toEqual({ mediaType: 'image/png', width: 1280, height: 800 }) + expect(imageDimensions(webpVP8X(1440, 900))).toEqual({ mediaType: 'image/webp', width: 1440, height: 900 }) + expect(() => imageDimensions(new Uint8Array(40))).toThrow('PNG or WebP') + }) +}) + +describe('validateListing', () => { + test('accepts the sample and enforces the contract', () => { + expect(() => validateListing(listing)).not.toThrow() + expect(() => validateListing({ locales: { ja: { tagline: 'x' } } })).toThrow('requires the "en" locale') + expect(() => validateListing({ ...listing, highlights: [{ locales: { en: { title: 'x' } }, screenshot: 'missing' }] })) + .toThrow('unknown screenshot "missing"') + expect(() => validateListing({ ...listing, screenshots: [{ ...listing.screenshots![0], size: 600_000 }] })).toThrow('bytes') + expect(() => validateListing({ ...listing, screenshots: [{ ...listing.screenshots![0], alt: { ja: 'x' } }] })).toThrow('requires the "en" locale') + expect(() => validateListing({ locales: { en: { tagline: 'x'.repeat(201) } } })).toThrow('exceeds 200') + }) +}) + +describe('loadListing', () => { + test('resolves screenshot files to digests, sizes and dimensions', async () => { + const dir = await mkdtemp(join(tmpdir(), 'listing-')) + await mkdir(join(dir, 'screenshots')) + const image = png(1200, 750) + await writeFile(join(dir, 'screenshots', 'people.png'), image) + await writeFile(join(dir, 'listing.yaml'), [ + 'locales:', ' en: { tagline: People, groups and profiles. }', ' ja: { tagline: メンバー一覧 }', + 'highlights:', ' - screenshot: people', ' locales: { en: { title: Search }, ja: { title: 検索 } }', + 'screenshots:', ' - name: people', ' file: people.png', ' alt: { en: People list, ja: メンバー一覧 }', + ].join('\n')) + const { listing: loaded, media } = await loadListing(pathToFileURL(`${dir}/`)) + const digest = `sha256:${createHash('sha256').update(image).digest('hex')}` + expect(media).toHaveLength(1) + expect(media[0].bytes).toEqual(image) + expect(loaded.screenshots).toEqual([{ name: 'people', digest, mediaType: 'image/png', size: 33, width: 1200, height: 750, alt: { en: 'People list', ja: 'メンバー一覧' } }]) + expect(loaded.highlights?.[0].screenshot).toBe('people') + }) + + test('rejects paths escaping catalog/screenshots', async () => { + const dir = await mkdtemp(join(tmpdir(), 'listing-')) + await writeFile(join(dir, 'listing.yaml'), 'locales: { en: { tagline: x } }\nscreenshots: [{ name: a, file: ../x.png, alt: { en: a } }]\n') + await expect(loadListing(pathToFileURL(`${dir}/`))).rejects.toThrow('relative path') + }) +}) + +describe('uploadMedia', () => { + test('PUTs each blob by hex digest and checks the stored descriptor', async () => { + const image = png(10, 10) + const digest = `sha256:${createHash('sha256').update(image).digest('hex')}` + const calls: Array<{ url: string; type: string }> = [] + const fetcher = async (url: string, init: RequestInit): Promise => { + calls.push({ url, type: (init.headers as Record)['content-type'] }) + return Response.json({ digest, mediaType: 'image/png', size: 33 }) + } + await uploadMedia([{ ...listing.screenshots![0], digest, bytes: image }], 'https://api.example', 'token', fetcher) + expect(calls).toEqual([{ url: `https://api.example/api/v1/catalog/media/${digest.slice(7)}`, type: 'image/png' }]) + const mismatch = async (): Promise => Response.json({ digest, mediaType: 'image/png', size: 1 }) + await expect(uploadMedia([{ ...listing.screenshots![0], digest, bytes: image }], 'https://api.example', 'token', mismatch)) + .rejects.toThrow('does not match') + }) +}) + +describe('listing in entries', () => { + const template = [ + 'apiVersion: appthrust.io/v1alpha1', 'kind: ApplicationStackDefinition', 'metadata: { name: llm }', + 'spec:', ' display: { title: Directory, description: People., category: collaborate, iconRef: { name: llm } }', + ' versions: []', ' updatePolicy: { default: pinned, allowed: [pinned, follow] }', + ].join('\n') + const digest = `sha256:${'b'.repeat(64)}` + + test('renderEntry carries the authored listing and otherwise preserves the stored one', () => { + const first = renderEntry(template, '0.2.0', digest, undefined, listing) + expect(first.listing).toEqual(listing) + const next = renderEntry(template, '0.2.1', `sha256:${'c'.repeat(64)}`, first) + expect(next.listing).toEqual(listing) + expect(renderEntry(template, '0.2.0', digest).listing).toBeUndefined() + }) + + test('renderListingUpdate replaces only the listing', () => { + const current: CatalogEntry = { ...renderEntry(template, '0.2.0', digest), publishedAt: '2026-11-01T00:00:00Z' } + const updated = renderListingUpdate(current, listing) + expect(updated).toEqual({ ...current, listing }) + expect(() => renderListingUpdate({ ...current, slug: 'other' }, listing)).toThrow('slug llm') + }) +}) diff --git a/hack/catalog-listing.ts b/hack/catalog-listing.ts new file mode 100644 index 0000000..12cc073 --- /dev/null +++ b/hack/catalog-listing.ts @@ -0,0 +1,201 @@ +// Catalog listing: the publisher-authored presentation of this app (localized +// copy, highlights, digest-pinned screenshots). Authored in catalog/listing.yaml +// with screenshot files under catalog/screenshots/, uploaded to the Cloud +// catalog media store, and submitted inside the catalog entry. The platform +// projects it onto ApplicationStackDefinition.spec.display and the Dashboard +// renders it on the Marketplace card and detail page. +// +// Bounds mirror the platform contract (pkg/appcatalog.ValidateListing): +// locales ≤ 16 with "en" required; tagline ≤ 200, title ≤ 100, overview ≤ 4000, +// notes ≤ 2000 per version (≤ 32 versions); highlights ≤ 6 (title ≤ 80, +// body ≤ 400); screenshots ≤ 8, each ≤ 512 KiB image/webp or image/png with +// alt ≤ 300 per locale. +import { createHash } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import { YAML } from 'bun' + +export const FALLBACK_LOCALE = 'en' +export const MAX_MEDIA_SIZE = 512 * 1024 + +export interface ListingLocale { + title?: string + tagline: string + overview?: string + notes?: Record +} + +export interface ListingHighlight { + locales: Record + screenshot?: string +} + +export interface ListingMedia { + name: string + digest: string + mediaType: 'image/webp' | 'image/png' + size: number + width: number + height: number + alt: Record +} + +export interface Listing { + locales: Record + highlights?: ListingHighlight[] + screenshots?: ListingMedia[] +} + +/** catalog/listing.yaml: screenshots name a file instead of a digest. */ +interface ListingSource { + locales: Record + highlights?: ListingHighlight[] + screenshots?: Array<{ name: string; file: string; alt: Record }> +} + +export interface MediaBlob extends ListingMedia { + bytes: Uint8Array +} + +const localeTag = /^[a-z]{2,3}$/ +const dnsLabel = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/ +const versionName = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/ + +function bound(value: string | undefined, max: number, what: string): void { + if (value !== undefined && value.length > max) throw new Error(`${what} exceeds ${max} characters`) +} + +function localizedText(text: Record | undefined, max: number, what: string): void { + if (!text || typeof text !== 'object') throw new Error(`${what} must map locales to text`) + if (!(FALLBACK_LOCALE in text)) throw new Error(`${what} requires the "${FALLBACK_LOCALE}" locale`) + if (Object.keys(text).length > 16) throw new Error(`${what} has more than 16 locales`) + for (const [tag, value] of Object.entries(text)) { + if (!localeTag.test(tag)) throw new Error(`${what} locale "${tag}" must be a BCP 47 primary tag`) + if (!value || value.length > max) throw new Error(`${what} locale "${tag}" must be 1-${max} characters`) + } +} + +export function validateListing(listing: Listing): void { + if (!listing.locales || typeof listing.locales !== 'object') throw new Error('listing.locales is required') + if (!(FALLBACK_LOCALE in listing.locales)) throw new Error(`listing requires the "${FALLBACK_LOCALE}" locale`) + if (Object.keys(listing.locales).length > 16) throw new Error('listing has more than 16 locales') + for (const [tag, locale] of Object.entries(listing.locales)) { + if (!localeTag.test(tag)) throw new Error(`listing locale "${tag}" must be a BCP 47 primary tag`) + if (typeof locale.tagline !== 'string' || !locale.tagline) throw new Error(`listing locale "${tag}" requires a tagline`) + bound(locale.title, 100, `locale ${tag} title`) + bound(locale.tagline, 200, `locale ${tag} tagline`) + bound(locale.overview, 4000, `locale ${tag} overview`) + const notes = locale.notes ?? {} + if (Object.keys(notes).length > 32) throw new Error(`locale ${tag} has more than 32 version notes`) + for (const [version, text] of Object.entries(notes)) { + if (!versionName.test(version)) throw new Error(`locale ${tag} notes version "${version}" is invalid`) + bound(text, 2000, `locale ${tag} notes for ${version}`) + } + } + const screenshots = listing.screenshots ?? [] + if (screenshots.length > 8) throw new Error('listing has more than 8 screenshots') + const names = new Set() + for (const media of screenshots) { + if (!dnsLabel.test(media.name) || media.name.length > 63 || names.has(media.name)) { + throw new Error(`screenshot name "${media.name}" is invalid or duplicated`) + } + names.add(media.name) + if (!/^sha256:[a-f0-9]{64}$/.test(media.digest)) throw new Error(`screenshot ${media.name} must pin a sha256 digest`) + if (media.mediaType !== 'image/webp' && media.mediaType !== 'image/png') { + throw new Error(`screenshot ${media.name} has unsupported media type ${media.mediaType}`) + } + if (!(media.size > 0 && media.size <= MAX_MEDIA_SIZE)) throw new Error(`screenshot ${media.name} must be 1-${MAX_MEDIA_SIZE} bytes`) + if (!(media.width > 0 && media.height > 0)) throw new Error(`screenshot ${media.name} has invalid dimensions`) + localizedText(media.alt, 300, `screenshot ${media.name} alt`) + } + const highlights = listing.highlights ?? [] + if (highlights.length > 6) throw new Error('listing has more than 6 highlights') + highlights.forEach((highlight, index) => { + if (!highlight.locales || !(FALLBACK_LOCALE in highlight.locales)) throw new Error(`highlight ${index} requires the "${FALLBACK_LOCALE}" locale`) + if (Object.keys(highlight.locales).length > 16) throw new Error(`highlight ${index} has more than 16 locales`) + for (const [tag, locale] of Object.entries(highlight.locales)) { + if (!localeTag.test(tag) || !locale.title || locale.title.length > 80 || (locale.body ?? '').length > 400) { + throw new Error(`highlight ${index} locale "${tag}" is invalid`) + } + } + if (highlight.screenshot !== undefined && !names.has(highlight.screenshot)) { + throw new Error(`highlight ${index} references unknown screenshot "${highlight.screenshot}"`) + } + }) +} + +/** Image dimensions from the PNG IHDR or WebP VP8/VP8L/VP8X header. */ +export function imageDimensions(bytes: Uint8Array): { mediaType: ListingMedia['mediaType']; width: number; height: number } { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + if (bytes.length >= 24 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) { + return { mediaType: 'image/png', width: view.getUint32(16), height: view.getUint32(20) } + } + if (bytes.length >= 30 && String.fromCharCode(...bytes.subarray(0, 4)) === 'RIFF' && String.fromCharCode(...bytes.subarray(8, 12)) === 'WEBP') { + const chunk = String.fromCharCode(...bytes.subarray(12, 16)) + if (chunk === 'VP8X') { + const width = 1 + (bytes[24] | (bytes[25] << 8) | (bytes[26] << 16)) + const height = 1 + (bytes[27] | (bytes[28] << 8) | (bytes[29] << 16)) + return { mediaType: 'image/webp', width, height } + } + if (chunk === 'VP8L') { + const b0 = bytes[21], b1 = bytes[22], b2 = bytes[23], b3 = bytes[24] + return { mediaType: 'image/webp', width: 1 + (((b1 & 0x3f) << 8) | b0), height: 1 + (((b3 & 0x0f) << 10) | (b2 << 2) | ((b1 & 0xc0) >> 6)) } + } + if (chunk === 'VP8 ') { + return { mediaType: 'image/webp', width: view.getUint16(26, true) & 0x3fff, height: view.getUint16(28, true) & 0x3fff } + } + } + throw new Error('screenshot must be a PNG or WebP image') +} + +/** Reads catalog/listing.yaml and its screenshot files; returns the listing with digests and the blobs to upload. */ +export async function loadListing(catalogDir: URL): Promise<{ listing: Listing; media: MediaBlob[] }> { + const source = YAML.parse(await readFile(new URL('listing.yaml', catalogDir), 'utf8')) as ListingSource + const media: MediaBlob[] = [] + for (const shot of source.screenshots ?? []) { + if (typeof shot.file !== 'string' || shot.file.includes('..') || shot.file.startsWith('/')) { + throw new Error(`screenshot ${shot.name} file must be a relative path under catalog/screenshots`) + } + const bytes = new Uint8Array(await readFile(new URL(`screenshots/${shot.file}`, catalogDir))) + const { mediaType, width, height } = imageDimensions(bytes) + media.push({ + name: shot.name, + digest: `sha256:${createHash('sha256').update(bytes).digest('hex')}`, + mediaType, + size: bytes.byteLength, + width, + height, + alt: shot.alt, + bytes, + }) + } + const listing: Listing = { + locales: source.locales, + ...(source.highlights ? { highlights: source.highlights } : {}), + ...(media.length ? { screenshots: media.map(({ bytes: _bytes, ...descriptor }) => descriptor) } : {}), + } + validateListing(listing) + return { listing, media } +} + +/** Idempotent immutable uploads; the Cloud verifies the body digest against the path. */ +export async function uploadMedia( + media: MediaBlob[], + baseURL: string, + token: string, + fetcher: (url: string, init: RequestInit) => Promise = fetch, +): Promise { + for (const blob of media) { + const response = await fetcher(`${baseURL}/api/v1/catalog/media/${blob.digest.slice(7)}`, { + method: 'PUT', + headers: { authorization: `Bearer ${token}`, 'content-type': blob.mediaType }, + body: blob.bytes as Uint8Array, + signal: AbortSignal.timeout(60_000), + redirect: 'error', + }) + if (response.status !== 200) throw new Error(`PUT /catalog/media/${blob.name} failed (${response.status})`) + const stored = await response.json() as { digest?: string; size?: number; mediaType?: string } + if (stored.digest !== blob.digest || stored.size !== blob.size || stored.mediaType !== blob.mediaType) { + throw new Error(`stored media ${blob.name} does not match the local file`) + } + } +} diff --git a/hack/publish-entry.ts b/hack/publish-entry.ts index 84d65af..856f736 100644 --- a/hack/publish-entry.ts +++ b/hack/publish-entry.ts @@ -3,6 +3,7 @@ import { execFile } from 'node:child_process' import { appendFile } from 'node:fs/promises' import { parseArgs, promisify } from 'node:util' import { YAML } from 'bun' +import { loadListing, type Listing, type MediaBlob, uploadMedia, validateListing } from './catalog-listing' const catalogRepository = 'ghcr.io/appthrust/catalog' const imageRepository = 'ghcr.io/appthrust/llm' @@ -25,6 +26,7 @@ export interface CatalogEntry { versions: CatalogVersion[] updatePolicy: { default: 'pinned' | 'follow'; allowed: Array<'pinned' | 'follow'> } display: { category?: string; icon?: string } + listing?: Listing publishedAt: string | null } @@ -48,7 +50,7 @@ export function validateRelease(version: string, digest: string): void { } } -export function renderEntry(template: string, version: string, digest: string, previous?: CatalogEntry): CatalogEntry { +export function renderEntry(template: string, version: string, digest: string, previous?: CatalogEntry, listing?: Listing): CatalogEntry { validateRelease(version, digest) const definition = YAML.parse(template) as Definition if (definition.apiVersion !== 'appthrust.io/v1alpha1' || definition.kind !== 'ApplicationStackDefinition' || definition.metadata.name !== 'llm') { @@ -81,16 +83,31 @@ export function renderEntry(template: string, version: string, digest: string, p default: true, }) definition.spec.versions = [...versions.values()] + if (listing) validateListing(listing) + const carried = listing ?? previous?.listing return { slug: 'llm', definitionYAML: YAML.stringify(definition), versions: definition.spec.versions, updatePolicy: definition.spec.updatePolicy, display: { category: definition.spec.display.category, icon: definition.spec.display.iconRef?.name }, + ...(carried ? { listing: carried } : {}), publishedAt: previous?.publishedAt ?? null, } } +/** + * Replaces only the listing of the entry the catalog already records. Versions, + * policy and display stay exactly as stored: a listing change never implies a + * release. Requires a published entry; the first tag publish carries the + * listing itself. + */ +export function renderListingUpdate(current: CatalogEntry, listing: Listing): CatalogEntry { + if (current.slug !== 'llm') throw new Error('current entry must have slug llm') + validateListing(listing) + return { ...current, listing } +} + export function catalogContainsRelease(index: unknown, version: string, digest: string): boolean { if (!index || typeof index !== 'object' || !('schemaVersion' in index) || index.schemaVersion !== 1 || !('entries' in index) || !Array.isArray(index.entries)) return false @@ -132,15 +149,46 @@ export interface CatalogPublication { state: 'queued' | 'snapshot' } -export async function submitEntry( - template: string, +/** + * The image digest the catalog already records for `version`, or null. A + * re-run of the publish workflow (for example after a failed Cloud signing + * dispatch) must reuse this image instead of building a new one: versions are + * immutable, so a second build with a different digest is rejected. + */ +export async function recordedDigest( version: string, - digest: string, baseURL: string, token: string, fetcher: (url: string, init: RequestInit) => Promise = fetch, -): Promise { - validateRelease(version, digest) +): Promise { + const response = await fetcher(`${baseURL}/api/v1/catalog/entries`, { + method: 'GET', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + signal: AbortSignal.timeout(30_000), + redirect: 'error', + }) + if (response.status !== 200) throw new Error(`GET /catalog/entries failed (${response.status})`) + const current = await response.json() as { data: CatalogEntry[] } + if (!Array.isArray(current.data)) throw new Error('catalog entries response must contain data[]') + const recorded = current.data.find(entry => entry.slug === 'llm')?.versions.find(entry => entry.name === version) + if (!recorded) return null + const image = recorded.images.web + if (typeof image !== 'string' || !image.startsWith(`${imageRepository}@`) || Object.keys(recorded.images).length !== 1) { + throw new Error(`version ${version} is recorded with an unexpected image set`) + } + const digest = image.slice(imageRepository.length + 1) + if (!digestPattern.test(digest)) throw new Error(`version ${version} is recorded with an invalid digest`) + return digest +} + +interface CatalogClient { + request(path: string, method: string, body?: unknown): Promise + entryRequest(path: string, method: string, body?: unknown): Promise + currentEntry(): Promise + publish(): Promise +} + +function catalogClient(baseURL: string, token: string, fetcher: (url: string, init: RequestInit) => Promise): CatalogClient { async function request(path: string, method: string, body?: unknown): Promise { return fetcher(`${baseURL}/api/v1/catalog/${path}`, { method, @@ -155,25 +203,102 @@ export async function submitEntry( if (response.status !== 200) throw new Error(`${method} /catalog/${path} failed (${response.status})`) return response.json() } - const current = await entryRequest('entries', 'GET') as { data: CatalogEntry[] } - if (!Array.isArray(current.data)) throw new Error('catalog entries response must contain data[]') - const entry = renderEntry(template, version, digest, current.data.find(entry => entry.slug === 'llm')) - await entryRequest('entries/llm', 'PUT', entry) - const response = await request('publish', 'POST') - if (response.status !== 200 && response.status !== 202) throw new Error(`POST /catalog/publish failed (${response.status})`) - const publication = await response.json() as CatalogPublication - if (typeof publication.id !== 'string' || !publication.id || /[\r\n]/.test(publication.id) || - !(response.status === 202 && publication.state === 'queued' || response.status === 200 && publication.state === 'snapshot')) { - throw new Error('catalog publish response must contain a queued (202) or snapshot (200) snapshot id') + return { + request, + entryRequest, + async currentEntry() { + const current = await entryRequest('entries', 'GET') as { data: CatalogEntry[] } + if (!Array.isArray(current.data)) throw new Error('catalog entries response must contain data[]') + return current.data.find(entry => entry.slug === 'llm') + }, + async publish() { + const response = await request('publish', 'POST') + if (response.status !== 200 && response.status !== 202) throw new Error(`POST /catalog/publish failed (${response.status})`) + const publication = await response.json() as CatalogPublication + if (typeof publication.id !== 'string' || !publication.id || /[\r\n]/.test(publication.id) || + !(response.status === 202 && publication.state === 'queued' || response.status === 200 && publication.state === 'snapshot')) { + throw new Error('catalog publish response must contain a queued (202) or snapshot (200) snapshot id') + } + return publication + }, } - return publication +} + +export async function submitEntry( + template: string, + version: string, + digest: string, + baseURL: string, + token: string, + fetcher: (url: string, init: RequestInit) => Promise = fetch, + listing?: { listing: Listing; media: MediaBlob[] }, +): Promise { + validateRelease(version, digest) + const client = catalogClient(baseURL, token, fetcher) + const previous = await client.currentEntry() + if (listing) await uploadMedia(listing.media, baseURL, token, fetcher) + const entry = renderEntry(template, version, digest, previous, listing?.listing) + await client.entryRequest('entries/llm', 'PUT', entry) + return client.publish() +} + +/** Uploads screenshots, replaces the stored entry's listing, and requests a publication. */ +export async function submitListing( + source: { listing: Listing; media: MediaBlob[] }, + baseURL: string, + token: string, + fetcher: (url: string, init: RequestInit) => Promise = fetch, +): Promise { + const client = catalogClient(baseURL, token, fetcher) + const current = await client.currentEntry() + if (!current) throw new Error('catalog has no llm entry yet; publish a release tag first') + await uploadMedia(source.media, baseURL, token, fetcher) + await client.entryRequest('entries/llm', 'PUT', renderListingUpdate(current, source.listing)) + return client.publish() +} + +async function optionalListing(): Promise<{ listing: Listing; media: MediaBlob[] } | undefined> { + const catalogDir = new URL('../catalog/', import.meta.url) + if (!(await Bun.file(new URL('listing.yaml', catalogDir)).exists())) return undefined + return loadListing(catalogDir) } async function main(): Promise { const [subcommand, ...args] = Bun.argv.slice(2) const { values } = parseArgs({ args, options: { version: { type: 'string' }, digest: { type: 'string' } }, strict: true }) + if (subcommand === 'recorded') { + if (!values.version || values.digest) throw new Error('usage: bun hack/publish-entry.ts recorded --version ') + validateRelease(values.version, `sha256:${'1'.repeat(64)}`) + const token = process.env.CLOUD_CATALOG_ADMIN_TOKEN + if (!token) throw new Error('CLOUD_CATALOG_ADMIN_TOKEN is required') + const baseURL = (process.env.CLOUD_API_URL ?? 'https://api.appthrust.com').replace(/\/$/, '') + const digest = await recordedDigest(values.version, baseURL, token) + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `digest=${digest ?? ''}\n`) + console.log(digest ? `Catalog already records ${values.version} as ${digest}` : `Catalog has no image for ${values.version}`) + return + } + if (subcommand === 'validate-listing') { + const source = await optionalListing() + if (!source) throw new Error('catalog/listing.yaml not found') + console.log(`Listing valid: ${Object.keys(source.listing.locales).join(',')} locales, ${source.listing.highlights?.length ?? 0} highlights, ${source.media.length} screenshots (${source.media.reduce((sum, media) => sum + media.size, 0)} bytes)`) + return + } + if (subcommand === 'listing') { + if (values.version || values.digest) throw new Error('usage: bun hack/publish-entry.ts listing') + const token = process.env.CLOUD_CATALOG_ADMIN_TOKEN + if (!token) throw new Error('CLOUD_CATALOG_ADMIN_TOKEN is required') + const baseURL = (process.env.CLOUD_API_URL ?? 'https://api.appthrust.com').replace(/\/$/, '') + const source = await optionalListing() + if (!source) throw new Error('catalog/listing.yaml not found') + const publication = await submitListing(source, baseURL, token) + if (process.env.GITHUB_OUTPUT) { + await appendFile(process.env.GITHUB_OUTPUT, `snapshot_id=${publication.id}\nstate=${publication.state}\n`) + } + console.log(`Catalog snapshot ${publication.id}: ${publication.state}`) + return + } if ((subcommand !== 'submit' && subcommand !== 'await') || !values.version || !values.digest) { - throw new Error('usage: bun hack/publish-entry.ts --version --digest ') + throw new Error('usage: bun hack/publish-entry.ts --version [--digest ]') } validateRelease(values.version, values.digest) if (subcommand === 'submit') { @@ -181,7 +306,7 @@ async function main(): Promise { if (!token) throw new Error('CLOUD_CATALOG_ADMIN_TOKEN is required') const baseURL = (process.env.CLOUD_API_URL ?? 'https://api.appthrust.com').replace(/\/$/, '') const template = await Bun.file(new URL('../catalog/entry.yaml', import.meta.url)).text() - const publication = await submitEntry(template, values.version, values.digest, baseURL, token) + const publication = await submitEntry(template, values.version, values.digest, baseURL, token, fetch, await optionalListing()) if (process.env.GITHUB_OUTPUT) { await appendFile(process.env.GITHUB_OUTPUT, `snapshot_id=${publication.id}\nstate=${publication.state}\n`) }