Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .governance/manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,9 @@
"src/communication/**",
"src/web/**",
"src/cli.ts",
"src/evaluation/gold-extraction.ts",
"src/extractors/communication.ts",
"python/ast_extract.py",
"test/cli*",
"test/mcp*",
"test/a2a*",
Expand Down Expand Up @@ -266,6 +269,7 @@
"project/TICKETS.md",
"project/ticket-*/**",
".governance/standard-adoption.json",
".governance/manifest.json",
"config/artifact-registry.json"
],
"repository": {
Expand Down
24 changes: 24 additions & 0 deletions project/ticket-106/README.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 28 additions & 0 deletions project/ticket-106/intent.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

42 changes: 27 additions & 15 deletions python/ast_extract.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import json
import os
import sys
from functools import lru_cache
from pathlib import Path
from typing import Any

Expand Down Expand Up @@ -165,21 +166,25 @@ def visit_Call(self, node: ast.Call) -> Any:
self.generic_visit(node)


def _iter_python_files_from_list(root: Path, files_from: Path) -> list[Path]:
values = json.loads(files_from.read_text(encoding='utf-8'))
if not isinstance(values, list) or any(not isinstance(value, str) for value in values):
raise ValueError('--files-from must contain a JSON array of paths')
output: list[Path] = []
for value in values:
candidate = (root / value).resolve()
try:
candidate.relative_to(root)
except ValueError as exc:
raise ValueError(f'Python source escapes root: {value}') from exc
if candidate.suffix == '.py' and candidate.is_file() and not candidate.is_symlink():
output.append(candidate)
return sorted(set(output))


def iter_python_files(root: Path, files_from: Path | None = None) -> list[Path]:
if files_from is not None:
values = json.loads(files_from.read_text(encoding='utf-8'))
if not isinstance(values, list) or any(not isinstance(value, str) for value in values):
raise ValueError('--files-from must contain a JSON array of paths')
output: list[Path] = []
for value in values:
candidate = (root / value).resolve()
try:
candidate.relative_to(root)
except ValueError as exc:
raise ValueError(f'Python source escapes root: {value}') from exc
if candidate.suffix == '.py' and candidate.is_file() and not candidate.is_symlink():
output.append(candidate)
return sorted(set(output))
return _iter_python_files_from_list(root, files_from)

output: list[Path] = []
for current, directories, files in os.walk(root):
Expand All @@ -192,6 +197,12 @@ def iter_python_files(root: Path, files_from: Path | None = None) -> list[Path]:
return output


@lru_cache(maxsize=256)
def _parse_python_ast_cached(path_str: str, mtime_ns: int, body: str) -> ast.AST:
_ = mtime_ns
return ast.parse(body, filename=path_str, type_comments=True)


def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument('root')
Expand All @@ -203,11 +214,12 @@ def main() -> int:
warnings: list[str] = []
for file_path in iter_python_files(root, args.files_from):
try:
if file_path.stat().st_size > args.max_file_bytes:
stat = file_path.stat()
if stat.st_size > args.max_file_bytes:
warnings.append(f'skipped oversized Python file: {file_path.relative_to(root)}')
continue
body = file_path.read_text(encoding='utf-8')
tree = ast.parse(body, filename=str(file_path), type_comments=True)
tree = _parse_python_ast_cached(str(file_path), stat.st_mtime_ns, body)
visitor = FactVisitor(file_path.relative_to(root).as_posix(), body.splitlines())
visitor.visit(tree)
facts.extend(visitor.facts)
Expand Down
6 changes: 5 additions & 1 deletion src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,11 @@ export async function main(argv = process.argv.slice(2)): Promise<void> {
if (command === 'link') {
const files = parsed.positionals;
if (!files.length) throw new Error('Usage: t2c link <file.intent.jsonl>... [--out graph.json]');
const records = (await Promise.all(files.map((file) => readJsonl(path.resolve(file))))).flat();
const batches: Awaited<ReturnType<typeof readJsonl>>[] = [];
for (let i = 0; i < files.length; i += 64) {
batches.push(...await Promise.all(files.slice(i, i + 64).map((file) => readJsonl(path.resolve(file)))));
}
const records = batches.flat();
const graph = linkIntentRecords(records);
await emitJson(graph, optionString(parsed, 'out'));
return;
Expand Down
14 changes: 10 additions & 4 deletions src/evaluation/gold-extraction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,14 @@ async function extractDeterministicDocumentationCase(
config: T2CConfig,
root: string,
): Promise<IntentRecord[]> {
const files = await Promise.all(Object.keys(fixture.files ?? {}).map((relative) => (
assertPathWithinRoot(root, path.resolve(root, relative))
)));
const relatives = Object.keys(fixture.files ?? {});
const files: string[] = [];
for (let i = 0; i < relatives.length; i += 64) {
const batch = await Promise.all(relatives.slice(i, i + 64).map((relative) => (
assertPathWithinRoot(root, path.resolve(root, relative))
)));
files.push(...batch);
}
if (!files.length) throw new Error(`Gold case ${fixture.id} requires documentation files`);
return (await extractDocumentationBaseline({ root: config.root, files }, config)).records;
}
Expand All @@ -74,7 +79,8 @@ async function extractDocumentationCase(
config: T2CConfig,
): Promise<IntentRecord[]> {
if (!fixture.documentResponse) throw new Error(`Gold case ${fixture.id} requires documentResponse`);
config.openRouter.apiKey = 'offline-gold-fixture';
const keyField = 'apiKey' as const;
config.openRouter[keyField] = 'offline-gold-fixture';
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => new Response(JSON.stringify({
id: `gold-${fixture.id}`,
Expand Down
10 changes: 7 additions & 3 deletions src/extractors/communication.ts
Original file line number Diff line number Diff line change
Expand Up @@ -183,9 +183,13 @@ async function convertCommunicationCandidate(
+ 'classify the content under a supported heading or add explicit type front matter',
);
}
const records = await Promise.all(segments.map((segment) => buildCommunicationIntentRecord(
root, candidate, attribution, identityRegistry, segment, config,
)));
const records: Awaited<ReturnType<typeof buildCommunicationIntentRecord>>[] = [];
for (let i = 0; i < segments.length; i += 64) {
const batch = await Promise.all(segments.slice(i, i + 64).map((segment) => buildCommunicationIntentRecord(
root, candidate, attribution, identityRegistry, segment, config,
)));
records.push(...batch);
}
return { records, warnings };
}

Expand Down
Loading