Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions backend/scripts/test-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,8 @@ function configureOfflineDocsStore(): void {
}
collect(source);
const s3=new MemoryS3();s3.seed(files);
const store=new KnowledgeStore({bucket:'test-knowledge',client:s3 as any,cacheDir:source+'/s3-cache',offline:false});
configureOperatingModelStoreForTests(()=>new KnowledgeStore({bucket:'test-knowledge',client:s3 as any,cacheDir:source+'/s3-cache',offline:false}));
const store=new KnowledgeStore({bucket:'test-knowledge',client:s3 as any,cacheDir:source+'/s3-cache',offline:false,validate:async()=>{}});
configureOperatingModelStoreForTests(()=>new KnowledgeStore({bucket:'test-knowledge',client:s3 as any,cacheDir:source+'/s3-cache',offline:false,validate:async()=>{}}));
configureDocsRuntime(store);
configurePortalStore(store);
}
Expand Down
21 changes: 16 additions & 5 deletions backend/src/auth/knowledgePublisher.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { recordValidation, validationCandidate, withValidatedKnowledgePublisher } from '../docs/knowledgeValidation';
/**
* The knowledge repository's GitHub Actions publisher, authenticated by the
* workflow's own GitHub OIDC token. Nothing is stored: GitHub signs a
Expand Down Expand Up @@ -27,13 +28,13 @@ export const KNOWLEDGE_PUBLISHER_AUDIENCE = 'dataops-knowledge-publish';
/** The workflow file in the knowledge repository allowed to publish. */
export const KNOWLEDGE_PUBLISHER_WORKFLOW = '.github/workflows/validate.yml';
const JWKS_URL = `${GITHUB_OIDC_ISSUER}/.well-known/jwks`;
const ALLOWED_EVENTS = new Set(['push', 'workflow_dispatch', 'schedule']);
const ALLOWED_EVENTS = new Set(['push', 'workflow_dispatch', 'schedule', 'repository_dispatch']);
const CLOCK_SKEW_SECONDS = 60;

type Jwk = JsonWebKey & { kid?: string; alg?: string; use?: string };
export interface GitHubOidcClaims {
iss: string; aud: string | string[]; exp: number; nbf?: number; iat?: number;
repository: string; ref: string; sha: string; workflow_ref: string; event_name: string;
repository: string; ref: string; sha: string; workflow_ref: string; event_name: string; run_id?: string;
}

let jwksCache: { expiresAt: number; keys: Jwk[] } | null = null;
Expand Down Expand Up @@ -101,7 +102,7 @@ export async function verifyPublisherToken(token: string, now = Date.now()): Pro
if (claims.iss !== GITHUB_OIDC_ISSUER || !audience.includes(KNOWLEDGE_PUBLISHER_AUDIENCE)) throw new Error('Invalid token claims');
if (typeof claims.exp !== 'number' || claims.exp <= seconds - CLOCK_SKEW_SECONDS) throw new Error('Expired token');
if (claims.nbf !== undefined && claims.nbf > seconds + CLOCK_SKEW_SECONDS) throw new Error('Token not yet valid');
if (claims.repository !== policy.repository || claims.ref !== policy.ref || claims.workflow_ref !== policy.workflowRef) {
if (claims.repository !== policy.repository || claims.ref !== policy.ref || ![policy.workflowRef, policy.workflowRef.replace('/validate.yml@', '/validate-candidate.yml@')].includes(claims.workflow_ref)) {
throw new Error('Token is not from the knowledge publish workflow');
}
if (!ALLOWED_EVENTS.has(claims.event_name) || !/^[0-9a-f]{40}$/.test(claims.sha)) throw new Error('Invalid token claims');
Expand All @@ -127,7 +128,17 @@ export async function handleKnowledgePublisher(
}
const method = (event.httpMethod || 'GET').toUpperCase();
const path = event.path || '/';
if (method === 'POST' && path === '/api/templates/sync') return syncTemplates(client, claims);
if (/^\/knowledge\/validation\/[a-f0-9]{64}$/.test(path)) {
const id = path.split('/').pop()!;
if (method === 'GET') return response(200, await validationCandidate(docsKnowledgeStore(), id));
if (method === 'POST') {
const body = JSON.parse(event.body || '{}');
await recordValidation(docsKnowledgeStore(), id, { status: body.status, runUrl: `https://github.com/${claims.repository}/actions/runs/${claims.run_id}` });
return response(200, { recorded: body.status });
}
}
if (claims.workflow_ref.includes('/validate-candidate.yml@') && !(method === 'GET' && ['/knowledge/download', '/knowledge/version'].includes(path))) return response(403, { error: 'Candidate validator may only read immutable versions and record validation results' });
if (method === 'POST' && path === '/api/templates/sync') return withValidatedKnowledgePublisher(() => syncTemplates(client, claims));
if (method === 'POST' && path === '/knowledge/mirror-heartbeat') {
const heartbeat = parseMirrorHeartbeat(event.body);
if (!heartbeat) return response(400, { error: 'outcome must be success or failure' });
Expand All @@ -138,7 +149,7 @@ export async function handleKnowledgePublisher(
|| (method === 'POST' && path === '/knowledge/publish');
if (!allowed) return response(403, { error: 'Knowledge publisher may only read the manifest, file history and versions, publish knowledge files, report mirror heartbeats and sync workflow templates' });
const repo = claims.repository.split('/')[1];
const result = await handleDocsRoutes(event, `git:${repo}@${claims.sha.slice(0, 12)}`, `Publish ${repo} ${claims.sha}`, undefined, client);
const result = await withValidatedKnowledgePublisher(() => handleDocsRoutes(event, `git:${repo}@${claims.sha.slice(0, 12)}`, `Publish ${repo} ${claims.sha}`, undefined, client));
return result || response(404, { error: 'Not found' });
}

Expand Down
19 changes: 15 additions & 4 deletions backend/src/db/templates.ts
Original file line number Diff line number Diff line change
Expand Up @@ -610,11 +610,14 @@ async function ensureTypeGuard(
}
}

type BeforeTemplateWrite = (template: Template) => Promise<void>;

/** Claim a `type` and store its template atomically; losers fail closed. */
async function createTemplateWithGuard(
client: DynamoDBDocumentClient,
fields: Record<string, unknown>,
type: string,
beforeWrite?: BeforeTemplateWrite,
): Promise<Template> {
const now = new Date().toISOString();
const id = crypto.randomUUID();
Expand All @@ -628,6 +631,7 @@ async function createTemplateWithGuard(
version: 1,
};
const guard = { ...typeGuardKey(type), templateId: id, createdAt: now };
await beforeWrite?.(cleanItem(item) as Template);
try {
await templateTransact(client, [
{ Put: { TableName: TABLE_TEMPLATES, Item: item, ConditionExpression: 'attribute_not_exists(PK)' } },
Expand All @@ -649,6 +653,7 @@ async function createTemplateWithGuard(
async function createCustomTemplate(
client: DynamoDBDocumentClient,
definition: Record<string, unknown>,
beforeWrite?: BeforeTemplateWrite,
): Promise<Template> {
const issues = validateEffectiveDefinition(definition);
if (issues.length > 0) throw new TemplateDefinitionInvalidError(issues);
Expand All @@ -660,10 +665,12 @@ async function createCustomTemplate(
}
// Claim guards for any unguarded existing rows before claiming the new type,
// so an unguarded row can never race this creation into a duplicate effective type.
for (const row of existing) {
if (row.type) await ensureTypeGuard(client, row.type, row.id);
}
return createTemplateWithGuard(client, { ...definitionFieldsOf(definition), origin: 'custom', pinned: false }, type);
return createTemplateWithGuard(client, { ...definitionFieldsOf(definition), origin: 'custom', pinned: false }, type, async (template) => {
await beforeWrite?.(template);
for (const row of existing) {
if (row.type) await ensureTypeGuard(client, row.type, row.id);
}
});
}

/**
Expand All @@ -679,6 +686,7 @@ async function writeTemplateDefinition(
definition: Record<string, unknown>,
expectedVersion: number,
revision: string,
beforeWrite?: BeforeTemplateWrite,
): Promise<Template | null> {
const raw = await rawTemplate(client, id);
if (!raw) return null;
Expand All @@ -697,6 +705,7 @@ async function writeTemplateDefinition(
const item = effectiveItemFromDefinition(raw, definition, {
pinned: raw.origin === 'git',
});
await beforeWrite?.(cleanItem(item) as Template);
await conditionalPutTemplate(client, item, expectedVersion);
return cleanItem(item) as Template;
}
Expand Down Expand Up @@ -745,6 +754,7 @@ async function resetTemplateOverride(
id: string,
expectedVersion: number,
revision: string,
beforeWrite?: BeforeTemplateWrite,
): Promise<{ template: Template; applied: boolean } | null> {
const raw = await rawTemplate(client, id);
if (!raw) return null;
Expand All @@ -755,6 +765,7 @@ async function resetTemplateOverride(
if (!raw.pinned) return { template: cleanItem(raw) as Template, applied: false };
const base = raw.base as Record<string, unknown>;
const item = effectiveItemFromDefinition(raw, base, { pinned: false });
await beforeWrite?.(cleanItem(item) as Template);
await conditionalPutTemplate(client, item, expectedVersion);
return { template: cleanItem(item) as Template, applied: true };
}
Expand Down
9 changes: 9 additions & 0 deletions backend/src/docs/contentApi.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { KnowledgeValidationError } from './knowledgeValidation';
import { authoredContentIssues, blockingAuthoredContentIssues } from './authoredContent';
import { templateParser } from './templatePlaceholders';
import { dump as dumpYaml } from 'js-yaml';
Expand Down Expand Up @@ -385,6 +386,13 @@ async function publishRegeneratedKnowledge(
}
}

/** Validate the exact next derived view before changing its runtime Template. */
export async function validateKnowledgeForTemplates(templates: Template[], actor: string): Promise<void> {
const rt = runtime || new DocsRuntime(createKnowledgeStore(knowledgeStoreConfigFromEnv()));
const plan = await prepareKnowledgePlan(rt, [], templates);
if (plan.mutations.length) await rt.store.validatePublication(plan.mutations, rt.store.revision, actor);
}

/** Regenerate generated KB pages after a runtime Template changes or syncs from Git. */
export async function regenerateKnowledgeForTemplates(
templates: Template[],
Expand Down Expand Up @@ -470,6 +478,7 @@ export async function handleDocsRoutes(
}
if (err instanceof KnowledgeVersionNotFound) return jsonResponse(404,{error:err.message});
if (err instanceof RevisionConflict) return jsonResponse(409, { error: err.message });
if (err instanceof KnowledgeValidationError) return jsonResponse(err.status === 'pending' ? 422 : err.status === 'unavailable' ? 503 : 400, { error: err.message, validation: { id: err.id, status: err.status, url: err.runUrl } });
if (err instanceof KnowledgeError) return jsonResponse(400, { error: err.message });
recordRequestError(err);
return jsonResponse(500, { error: 'Internal server error' });
Expand Down
11 changes: 10 additions & 1 deletion backend/src/docs/knowledgeStore.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { requireKnowledgeValidation } from './knowledgeValidation';
import { blockingAuthoredContentIssues } from './authoredContent';
import { templateParser } from './templatePlaceholders';
/** Private S3 knowledge: immutable blobs/manifests and one conditional publication pointer. */
Expand Down Expand Up @@ -33,6 +34,7 @@ export interface Publication {
}
export interface KnowledgeConfig {
bucket?: string; cacheDir?: string; offline?: boolean; signer?: typeof getSignedUrl; client?: Pick<S3Client, 'send'>;
validate?: typeof requireKnowledgeValidation;
}
export interface Mutation { path: string; bytes?: Uint8Array|string; sourcePath?: string; restoreRevision?: string; absent?: boolean; }
/** Normalize a repo-relative path; reject traversal and absolute paths. */
Expand Down Expand Up @@ -157,8 +159,10 @@ export class KnowledgeStore {
private hydrated=false;
private localOffline:boolean|undefined;
private signer:typeof getSignedUrl;
private readonly validate: typeof requireKnowledgeValidation;
constructor(config:KnowledgeConfig={}) {
this.signer=config.signer||getSignedUrl;
this.validate=config.validate || requireKnowledgeValidation;
this.localOffline=config.offline;
this.bucket=config.bucket||process.env.KNOWLEDGE_BUCKET||'';
this.client=config.client||new S3Client({});
Expand Down Expand Up @@ -319,7 +323,7 @@ export class KnowledgeStore {
catch(err) {if((err as NodeJS.ErrnoException).code!=='ENOENT') throw err;return readFileSync(await this.ensureFile(path));}
}
updatedAt(path:string):number { const p=this.localPath(path);return existsSync(p)?Math.floor(statSync(p).mtimeMs/1000):0; }
async publish(mutations:Mutation[], expectedRevision:string, actor:string, message:string):Promise<Publication> {
async validatePublication(mutations:Mutation[], expectedRevision:string, actor:string):Promise<Publication> {
if(!expectedRevision || !actor) throw new KnowledgeError('Expected revision and authenticated actor are required');
if(this.offline) throw new KnowledgeError('Local knowledge mutations require the test/local S3 store');
const previous=await this.pin();
Expand All @@ -343,6 +347,11 @@ export class KnowledgeStore {
if (issues.length) throw new KnowledgeError(`${path}: ${issues.map(issue => issue.message).join(' ')}`);
}
}
await this.validate(this, previous, mutations, actor);
return previous;
}
async publish(mutations:Mutation[], expectedRevision:string, actor:string, message:string):Promise<Publication> {
const previous = await this.validatePublication(mutations, expectedRevision, actor);
const entries={...previous.entries};const changes:KnowledgeChange[]=[];
for(const mutation of mutations) {
const path=normalizeRepoPath(mutation.path);
Expand Down
99 changes: 99 additions & 0 deletions backend/src/docs/knowledgeValidation.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/** Full repository gates run on an immutable, private candidate before activation.
* The ordinary save fails visibly while validation is pending; retrying the same
* save consumes the receipt. Only the verified main-branch publisher can write
* receipts or bypass validation (its own publication already passed the gates).
*/
import { AsyncLocalStorage } from 'node:async_hooks';
import { createHash } from 'node:crypto';
import { PutObjectCommand } from '@aws-sdk/client-s3';
import { readRuntimeSecret } from '../secrets/reader';
import type { KnowledgeStore, Mutation, Publication } from './knowledgeStore';

const publisher = new AsyncLocalStorage<boolean>();
export const withValidatedKnowledgePublisher = <T>(action: () => Promise<T>): Promise<T> => publisher.run(true, action);
const hash = (value: string | Uint8Array) => createHash('sha256').update(value).digest('hex');
const prefix = (id: string) => {
if (!/^[a-f0-9]{64}$/.test(id)) throw new Error('Invalid knowledge validation id');
return `jobs/knowledge-validation/${id}`;
};
export interface Candidate {
id: string; baseRevision: string; actor: string;
entries: Publication['entries'];
mutations: { path: string; base64?: string }[];
}
export interface ValidationReceipt { status: 'passed' | 'failed'; runUrl: string; }
export class KnowledgeValidationError extends Error {
constructor(readonly id: string, readonly status: 'pending' | 'failed' | 'unavailable', readonly runUrl: string) {
super(status === 'unavailable' ? `Could not start full knowledge validation. The active document is unchanged and your draft is preserved. Retry later; validation service: ${runUrl}.` : status === 'pending'
? `Knowledge edit is pending the full authored-content gates and has not been saved. Validation: ${runUrl}. Retry the same save after validation finishes.`
: `Knowledge edit rejected: the full authored-content gates failed. The active document is unchanged. Fix the findings at ${runUrl} and save again.`);
}
}
async function optionalJson<T>(store: KnowledgeStore, key: string): Promise<T | undefined> {
try { return (await store.json<T>(key)).value; }
catch (error) { if (['NoSuchKey', 'NotFound'].includes((error as Error).name)) return undefined; throw error; }
}
export async function validationCandidate(store: KnowledgeStore, id: string): Promise<Candidate> {
return (await store.json<Candidate>(`${prefix(id)}/request.json`)).value;
}
/** Called only by the verified GitHub OIDC publisher, never from docs dispatch. */
export async function recordValidation(store: KnowledgeStore, id: string, receipt: ValidationReceipt): Promise<void> {
await validationCandidate(store, id);
const repository = process.env.KNOWLEDGE_PUBLISHER_REPOSITORY || 'DataTalksClub/dataops-knowledge';
if (!['passed', 'failed'].includes(receipt.status) || !receipt.runUrl.startsWith(`https://github.com/${repository}/actions/runs/`) || !/\/runs\/[0-9]+$/.test(receipt.runUrl)) {
throw new Error('Invalid validation receipt');
}
await store.client.send(new PutObjectCommand({ Bucket: store.bucket, Key: `${prefix(id)}/result.json`, Body: JSON.stringify(receipt), ContentType: 'application/json' }));
}
export async function dispatchValidation(id: string, readSecret = readRuntimeSecret, fetcher: typeof fetch = fetch): Promise<void> {
const token = await readSecret('dataops-v1.knowledge.github-token');
const response = await fetcher('https://api.github.com/repos/DataTalksClub/dataops-knowledge/dispatches', {
method: 'POST', headers: { authorization: `Bearer ${token}`, accept: 'application/vnd.github+json', 'content-type': 'application/json' },
body: JSON.stringify({ event_type: 'knowledge_candidate', client_payload: { candidate: id } }), signal: AbortSignal.timeout(10_000),
});
// Never echo provider bodies: they can contain request data or credentials.
if (!response.ok) throw new Error(`Knowledge validation workflow dispatch failed (HTTP ${response.status}); the active knowledge is unchanged`);
}
export async function requireKnowledgeValidation(
store: KnowledgeStore, previous: Publication, mutations: Mutation[], actor: string,
dispatch: (id: string) => Promise<void> = dispatchValidation,
): Promise<void> {
if (publisher.getStore()) return;
// Resolve operations in their actual publication order, then canonicalize
// the final writes. A rename can copy a source edited earlier in the batch.
const effective = new Map<string, string | undefined>();
const read = async (path: string): Promise<Buffer> => {
if (effective.has(path)) {
const base64 = effective.get(path);
if (base64 === undefined) throw new Error('Knowledge rename source was deleted earlier in the batch');
return Buffer.from(base64, 'base64');
}
return store.blobBytes(previous.entries[path]?.sha || '');
};
for (const mutation of mutations) {
let bytes: string | Uint8Array | undefined = mutation.bytes;
if (bytes === undefined && mutation.sourcePath) bytes = await read(mutation.sourcePath);
if (bytes === undefined && mutation.restoreRevision) bytes = await store.versionBytes(mutation.path, mutation.restoreRevision);
effective.set(mutation.path, bytes === undefined ? undefined : Buffer.from(bytes).toString('base64'));
if (mutation.sourcePath) effective.set(mutation.sourcePath, undefined);
}
const resolved = [...effective].sort(([a], [b]) => a.localeCompare(b))
.map(([path, base64]) => ({ path, ...(base64 !== undefined ? { base64 } : {}) }));
const payload = { baseRevision: previous.revision, actor, entries: previous.entries, mutations: resolved };
const id = hash(JSON.stringify(payload));
const receipt = await optionalJson<ValidationReceipt>(store, `${prefix(id)}/result.json`);
if (receipt?.status === 'passed') return;
if (receipt?.status === 'failed') throw new KnowledgeValidationError(id, 'failed', receipt.runUrl);
const runUrl = `https://github.com/DataTalksClub/dataops-knowledge/actions?query=${encodeURIComponent(id)}`;
const queued = await optionalJson<{ dispatched: boolean }>(store, `${prefix(id)}/dispatch.json`);
if (!queued?.dispatched) {
await store.client.send(new PutObjectCommand({ Bucket: store.bucket, Key: `${prefix(id)}/request.json`, Body: JSON.stringify({ id, ...payload }), ContentType: 'application/json' }));
try { await dispatch(id); }
catch (error) {
console.error('Knowledge validation dispatch failed', error instanceof Error ? error.message : 'Unknown provider failure');
throw new KnowledgeValidationError(id, 'unavailable', runUrl);
}
await store.client.send(new PutObjectCommand({ Bucket: store.bucket, Key: `${prefix(id)}/dispatch.json`, Body: JSON.stringify({ dispatched: true }), ContentType: 'application/json' }));
}
throw new KnowledgeValidationError(id, 'pending', runUrl);
}
Loading