Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions .github/workflows/auto-node-docs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: Zero-code Node docs

on:
pull_request:
branches: [main, dev]
paths:
- 'src/pages/docs/integrations/traceai/auto.mdx'
- 'scripts/auto-*.mjs'
- '.github/workflows/auto-node-docs.yml'
workflow_dispatch:

permissions:
contents: read

jobs:
published-snippets:
name: Published npm snippets (Node ${{ matrix.node }})
runs-on: ubuntu-24.04
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
node: ['20.6.0', '22.18.0']
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node }}
- name: Audit public reference links and heading anchors
run: node scripts/auto-links.mjs
- name: Run every guide snippet with loopback provider and OTLP
run: node scripts/auto-snippets.mjs
- uses: actions/upload-artifact@v4
if: always()
with:
name: auto-snippets-${{ matrix.node }}
path: /tmp/docs-auto-evidence
if-no-files-found: error
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"new-doc": "node scripts/new-doc.mjs",
"audit-links": "node scripts/audit-links.mjs",
"test:integrations": "node --test scripts/integrations.test.mjs scripts/new-doc.test.mjs scripts/langfuse-import.test.mjs scripts/n8n-page.test.mjs scripts/openlit-endpoint.test.mjs scripts/chromadb-java.test.mjs scripts/openllmetry-content.test.mjs",
"test:integrations:browser": "node scripts/integrations.browser.cjs && node scripts/integrations.regression.cjs && node scripts/widget-regression.cjs",
"test:integrations:browser": "node scripts/browser-network.browser.cjs && node scripts/integrations.browser.cjs && node scripts/integrations.regression.cjs && node scripts/widget-regression.cjs && node scripts/auto.browser.mjs",
"check:integrations": "tsc --allowJs --checkJs --noEmit --strict --types node --target ES2022 --module nodenext src/lib/integrations.mjs"
},
"dependencies": {
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
23 changes: 23 additions & 0 deletions scripts/auto-links.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Audit public package references, not signed-in application routes.
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
const page = await readFile(new URL('../src/pages/docs/integrations/traceai/auto.mdx', import.meta.url), 'utf8');
const urls = new Set([...page.matchAll(/\]\((https:\/\/github\.com\/[^)]+)\)/g)].map(match => match[1]));
const documents = new Map();
for (const href of urls) {
const url = new URL(href);
const anchor = url.hash.slice(1);
url.hash = '';
const source = url.href.replace('https://github.com/future-agi/traceAI/blob/', 'https://raw.githubusercontent.com/future-agi/traceAI/');
if (!documents.has(source)) {
const response = await fetch(source, { signal: AbortSignal.timeout(30000) });
assert.ok(response.ok, `${source}: ${response.status}`);
documents.set(source, await response.text());
}
if (anchor && source.startsWith('https://raw.githubusercontent.com/')) {
const headings = [...documents.get(source).matchAll(/^#{1,6} (.+)$/gm)].map(match => match[1].toLowerCase().replace(/[^\p{L}\p{N}\s_-]/gu, '').replace(/ /g, '-'));
assert.ok(headings.includes(anchor), `${href}: missing heading`);
}
console.log(`OK ${href}`);
}
console.log(JSON.stringify({ externalLinks: urls.size, fetchedDocuments: documents.size }));
59 changes: 59 additions & 0 deletions scripts/auto-page.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync, existsSync } from 'node:fs';
import { integrations, integrationNavigation } from '../src/lib/integrations.mjs';

const read = path => readFileSync(new URL(`../${path}`, import.meta.url), 'utf8');
const page = read('src/pages/docs/integrations/traceai/auto.mdx');
const route = '/docs/integrations/traceai/auto';

test('zero-code Node guide is discoverable from the catalog, sidebar and setup paths', () => {
const entry = integrations.find(item => item.id === 'traceai-auto');
assert.equal(entry?.href, route);
assert.equal(entry?.category, 'frameworks');
assert.ok(entry?.aliases.includes('@traceai/auto'));
assert.ok(JSON.stringify(integrationNavigation).includes(route));
for (const file of ['quickstart/setup-observability', 'integrations/traceai/index']) {
assert.ok(read(`src/pages/docs/${file}.mdx`).includes(route));
}
});

test('primary setup preloads a maintained package without app instrumentation', () => {
assert.match(page, /Node\.js 20\.6 or later/);
assert.match(page, /https:\/\/app\.futureagi\.com\/dashboard\/keys/);
assert.match(page, /<Steps>[\s\S]*<Step title="Install the package">/);
for (const command of ['npm install @traceai/auto', 'node --import @traceai/auto/register app.mjs', 'NODE_OPTIONS="--import @traceai/auto/register" npm start', 'node --require @traceai/auto/register app.cjs', 'npx --no-install traceai-auto doctor']) {
assert.ok(page.includes(command), command);
}
const examples = [...page.matchAll(/```javascript\n([\s\S]*?)\n```/g)];
assert.equal(examples.length, 1);
assert.match(examples[0][1], /import OpenAI from "openai"/);
assert.doesNotMatch(examples[0][1], /traceai|opentelemetry|instrument|register/);
assert.match(page, /Observe → my-ai-app/);
assert.match(page, /zero exit code alone does not prove export health/);
});

test('HTTP scope distinguishes outgoing calls from the optional incoming integration', () => {
assert.match(page, /outgoing non-AI HTTP calls are not automatically traced/);
assert.match(page, /optional `@opentelemetry\/instrumentation-http` package is installed/);
assert.match(page, /incoming HTTP server spans/);
assert.match(page, /`FI_DISABLED_INSTRUMENTATIONS=http` to disable that integration/);
});

test('privacy defaults, runtime boundaries and reference links remain explicit', () => {
const setup = page.split('## Try a minimal')[0];
assert.match(setup, /captured by default/);
for (const flag of ['FI_HIDE_INPUTS=true', 'FI_HIDE_OUTPUTS=true', 'FI_HIDE_ERROR_DETAILS=true']) assert.ok(setup.includes(flag));
assert.match(page, /PII pattern redaction[^\n]*Off by default/);
assert.match(page, /raising it does not enable base64 image export/);
assert.match(page, /Other exporters retain their own privacy policies/);
assert.match(page, /README\.md#library-coverage/);
for (const anchor of ['nextjs', 'node-server-bundlers', 'aws-lambda-and-short-lived-processes', 'edge-and-browsers']) assert.ok(page.includes(`FRAMEWORKS.md#${anchor}`));
assert.match(page, /application-owned exporter/);
assert.match(page, /\/docs\/sdk\/tracing\/set-up-tracing/);
assert.doesNotMatch(page, /verified live|PASS\/FAIL|loopback|test harness|E2B|not tested/);
const image = page.match(/!\[[^\]]+\]\((\/images\/[^)]+)\)/)?.[1];
assert.ok(image);
assert.ok(existsSync(new URL(`../public${image}`, import.meta.url)));
assert.match(page, /OpenAI SDK 7 with openai\/gpt-4\.1-mini through OpenRouter/);
});
185 changes: 185 additions & 0 deletions scripts/auto-snippets.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
// Execute every fenced block in the guide against published npm packages.
// Run only in a disposable E2B sandbox or CI. No real service credentials needed.
import assert from 'node:assert/strict';
import { mkdtemp, mkdir, readFile, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { createServer } from 'node:http';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { gunzipSync } from 'node:zlib';

const exec = promisify(execFile);
const page = await readFile(new URL('../src/pages/docs/integrations/traceai/auto.mdx', import.meta.url), 'utf8');
const blocks = [...page.matchAll(/```(bash|javascript)\n([\s\S]*?)\n```/g)].map((m, index) => ({ index, language: m[1], code: m[2] }));
assert.equal(blocks.length, 10, 'Every new fenced block needs an execution case');
const block = prefix => {
const found = blocks.find(b => b.code.startsWith(prefix));
assert.ok(found, prefix);
return found;
};
const done = new Set();
const results = [];
const batches = [];
const requests = [];
const errors = [];
let collectorStatus = 200;
const app = await mkdtemp(join(tmpdir(), 'auto-docs-'));
const output = resolve(process.env.AUTO_EVIDENCE_DIR || '/tmp/docs-auto-evidence');
await mkdir(output, { recursive: true });
const server = createServer(async (req, res) => {
try {
const parts = [];
for await (const part of req) parts.push(part);
let raw = Buffer.concat(parts);
if (req.headers['content-encoding'] === 'gzip') raw = gunzipSync(raw);
const data = JSON.parse(raw.toString() || '{}');
if (req.url === '/v1/chat/completions') {
assert.equal(req.headers.authorization, 'Bearer synthetic-provider-key');
assert.equal(data.model, 'docs-test-model');
assert.equal(data.messages[0].content, 'Say hello in one word.');
requests.push(data);
res.writeHead(200, { 'content-type': 'application/json' });
res.end(JSON.stringify({ id: 'chatcmpl-docs', object: 'chat.completion', created: 1700000000, model: data.model, choices: [{ index: 0, message: { role: 'assistant', content: 'Hello!' }, finish_reason: 'stop' }], usage: { prompt_tokens: 13, completion_tokens: 3, total_tokens: 16 } }));
} else if (req.url === '/tracer/v1/traces') {
assert.equal(req.headers['x-api-key'], 'synthetic-fi-api-key');
assert.equal(req.headers['x-secret-key'], 'synthetic-fi-secret-key');
batches.push(data);
res.writeHead(collectorStatus, { 'content-type': 'application/json' });
res.end('{}');
} else {
throw new Error(`Unexpected request route: ${req.url}`);
}
} catch (error) {
errors.push(error.message);
res.writeHead(500, { 'content-type': 'application/json' });
res.end('{}');
}
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const url = `http://127.0.0.1:${server.address().port}`;
// Do not inherit FI credentials, provider keys, preload flags or user profiles.
const env = {
PATH: process.env.PATH,
HOME: join(app, 'home'),
XDG_CONFIG_HOME: join(app, 'config'),
FI_BASE_URL: url,
OPENAI_BASE_URL: `${url}/v1`,
OPENAI_API_KEY: 'synthetic-provider-key',
OPENAI_MODEL: 'docs-test-model',
npm_config_registry: 'https://registry.npmjs.org',
npm_config_update_notifier: 'false',
npm_config_audit: 'false',
npm_config_fund: 'false',
};
const setup = block('export FI_API_KEY');
const config = setup.code.replaceAll('your-api-key', 'synthetic-fi-api-key').replaceAll('your-secret-key', 'synthetic-fi-secret-key');
const allSpans = from => batches.slice(from).flatMap(batch => batch.resourceSpans.flatMap(resource => resource.scopeSpans.flatMap(scope => scope.spans)));
async function run(name, command, { expected = 'model', hidden = false, overrides = {} } = {}) {
const from = batches.length;
const before = requests.length;
const result = await exec('bash', ['-c', `set -euo pipefail\n${config}\n${command}`], { cwd: app, env: { ...env, ...overrides }, timeout: 180000, maxBuffer: 4 * 1024 * 1024 });
const spans = allSpans(from);
assert.deepEqual(errors, []);
if (expected === 'model') {
assert.equal(requests.length - before, 1, `${name}: provider calls`);
assert.match(result.stdout, /Hello!/);
assert.equal(spans.length, 1, `${name}: exactly one model span, no duplicate wire span`);
const serialized = JSON.stringify(spans);
const attributes = Object.fromEntries(spans[0].attributes.map(attribute => [attribute.key, attribute.value]));
assert.equal(attributes['gen_ai.request.model'].stringValue, 'docs-test-model');
assert.equal(Number(attributes['gen_ai.usage.input_tokens'].intValue), 13);
assert.equal(Number(attributes['gen_ai.usage.output_tokens'].intValue), 3);
assert.equal(Number(attributes['gen_ai.usage.total_tokens'].intValue), 16);
assert.ok(BigInt(spans[0].endTimeUnixNano) > BigInt(spans[0].startTimeUnixNano));
assert.match(JSON.stringify(batches.slice(from)), /my-ai-app/);
if (hidden) assert.doesNotMatch(serialized, /Say hello in one word\.|Hello!/);
else { assert.match(serialized, /Say hello in one word\./); assert.match(serialized, /Hello!/); }
assert.doesNotMatch(serialized, /synthetic-(?:fi|provider)/);
} else if (expected === 'http-server' || expected === 'no-http') {
assert.equal(requests.length - before, 0, `${name}: no AI provider call`);
assert.match(result.stdout, /ordinary health response/);
assert.equal(spans.length, expected === 'http-server' ? 1 : 0, `${name}: HTTP span count`);
if (expected === 'http-server') {
assert.equal(spans[0].kind, 2, 'Incoming request exports a SERVER span, not a CLIENT span');
assert.match(JSON.stringify(spans[0].attributes), /\/health/);
}
} else if (expected === 'doctor') {
const report = JSON.parse(result.stdout);
assert.equal(report.active, true);
assert.equal(report.collector, collectorStatus === 200 ? 'accepted' : 'failed');
assert.equal(spans.length, 1);
assert.equal(spans[0].name, 'traceai.auto.doctor');
} else if (expected === 'bootstrap') {
const report = JSON.parse(result.stdout);
assert.ok(report.integrations.some(i => i.name === 'openai' || i.id === 'openai'));
assert.ok(Array.isArray(report.install));
assert.equal(spans.length, 0);
}
results.push({ name, spans: spans.length, providerCalls: requests.length - before, exitCode: 0 });
console.log(JSON.stringify(results.at(-1)));
return result;
}
async function execute(prefix, options) {
const selected = block(prefix);
done.add(selected.index);
return run(prefix.split('\n')[0], selected.code, options);
}
try {
await mkdir(env.HOME, { recursive: true });
await writeFile(join(app, 'package.json'), JSON.stringify({ name: 'auto-docs-example', private: true, scripts: { start: 'node app.mjs' } }));
await execute('npm install @traceai/auto', { expected: 'install' });
await execute('npm install openai', { expected: 'install' });
const example = block('import OpenAI');
assert.doesNotMatch(example.code, /traceai|opentelemetry/);
await writeFile(join(app, 'app.mjs'), example.code);
// The CommonJS startup variant uses an equivalent normal application, no tracing imports.
await writeFile(join(app, 'app.cjs'), `const OpenAI = require('openai');\n(async () => {\n${example.code.split('\n').slice(1).join('\n')}\n})().catch(error => { console.error(error); process.exitCode = 1; });\n`);
done.add(example.index);
done.add(setup.index); // Executed before every shell command with synthetic substitutions.
await execute('node --import');
await execute('NODE_OPTIONS=');
await execute('node --require');
await execute('npx --no-install traceai-auto bootstrap', { expected: 'bootstrap' });
await execute('FI_HIDE_INPUTS=', { hidden: true });
await execute('npx --no-install traceai-auto doctor', { expected: 'doctor' });
// Regression: doctor exit code can be zero even when collection fails.
collectorStatus = 400;
await run('doctor rejects collector errors despite exit zero', block('npx --no-install traceai-auto doctor').code, { expected: 'doctor' });
collectorStatus = 200;
assert.equal(done.size, blocks.length, 'All page blocks executed');
// Ordinary HTTP is not AI traffic. The optional integration traces only the
// incoming half of this request; no manual tracing APIs are used in the app.
await writeFile(join(app, 'http.mjs'), `import { createServer, get } from 'node:http';
const server = createServer((request, response) => response.end('ordinary health response'));
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
try {
const body = await new Promise((resolve, reject) => {
get('http://127.0.0.1:' + server.address().port + '/health', response => {
let text = '';
response.setEncoding('utf8');
response.on('data', chunk => { text += chunk; });
response.on('end', () => resolve(text));
response.on('error', reject);
}).on('error', reject);
});
console.log(body);
} finally { await new Promise(resolve => server.close(resolve)); }
`);
const httpCommand = block('node --import').code.replace('app.mjs', 'http.mjs');
await run('HTTP without the optional instrumentor', httpCommand, { expected: 'no-http' });
await run('install optional HTTP instrumentor', 'npm install @opentelemetry/instrumentation-http@0.202.0', { expected: 'install' });
await run('HTTP integration captures incoming but not outgoing requests', httpCommand, { expected: 'http-server' });
assert.ok(page.includes('FI_DISABLED_INSTRUMENTATIONS=http'));
await run('HTTP denylist disables incoming spans', `FI_DISABLED_INSTRUMENTATIONS=http ${httpCommand}`, { expected: 'no-http' });
const packages = {};
for (const name of ['@traceai/auto', 'openai', '@opentelemetry/instrumentation-http']) packages[name] = JSON.parse(await readFile(join(app, 'node_modules', name, 'package.json'), 'utf8')).version;
const receipt = { node: process.version, packages, executedBlocks: done.size, totalBlocks: blocks.length, results, scope: 'Synthetic loopback provider and OTLP only; not a live Cloud test.' };
await writeFile(join(output, 'snippets.json'), JSON.stringify(receipt, null, 2));
await writeFile(join(output, 'otlp.json'), JSON.stringify(batches, null, 2));
await writeFile(join(output, 'package-lock.json'), await readFile(join(app, 'package-lock.json')));
console.log(JSON.stringify(receipt));
} finally {
await new Promise(resolve => server.close(resolve));
await rm(app, { recursive: true, force: true });
}
Loading
Loading