diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b07833e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: CI + +on: + pull_request: + branches: [release/0.x] + push: + branches: [release/0.x] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Typecheck + run: bun run typecheck + + - name: Unit + conformance tests + run: bun test + + # Container suite is opt-in; ubuntu-latest ships Docker, so testcontainers works out of the box. + - name: Neo4j bolt container tests + run: RUN_CONTAINER_TESTS=1 bun test test/neo4j-bolt.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..7ca4822 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,75 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.6.0] - 2026-08-05 + +### Added +- **JavaScript is analyzed** (#84). Discovery was restricted to `.ts/.tsx/.mts/.cts`, + so a JavaScript-only project produced an empty symbol table and exited 0 with no + warning — on OWASP NodeGoat, 0 modules and an 84-byte `analysis.json`. `.js`, + `.jsx`, `.mjs` and `.cjs` are now discovered, and `.test.js` / `.spec.js` are + skipped like their TypeScript counterparts. Nothing downstream needed changing: + the compiler already ran with `allowJs`, and Jelly already accepted `.js` — both + were simply never handed a file. +- **Methods declared through dynamic idioms are materialized** (#85): + `this. = fn` inside a constructor function, and object-literal members + (`{ foo(){} }`, `{ foo: function(){} }`). Previously the first landed in + `local_variables` and the second was dropped entirely, so no call could resolve + to either — call-graph edges are gated to signatures present in the symbol table. + This is language-neutral: both were missed in TypeScript too. + +### Changed +- **BREAKING: Neo4j labels and relationship types are namespaced per source language** (#88). + Node labels gain a language twin — a `.js` module is `:Module:JSModule`, a `.ts` module is + `:Module:TSModule` — and every relationship type is prefixed: `JS_CALLS`, `TS_DECLARES`, + `JS_HAS_MODULE`, and so on. This matches `codeanalyzer-python`, which already namespaces every + edge (`PY_CALLS`, `PY_DECLARES`, …), so a database holding output from more than one analyzer no + longer mingles them. + + An edge takes its **source** module's language, falling back to its target's — so the + application-to-module edge on a JavaScript project is `JS_HAS_MODULE`. Nodes with no language of + their own (the application root, packages, external library symbols) keep the analyzer's own `TS` + namespace, since a sibling analyzer emits its own. + + **Migration:** every stored query against a graph produced by 0.5.0 or earlier must be updated — + `MATCH ()-[:CALLS]->()` becomes `MATCH ()-[:TS_CALLS|JS_CALLS]->()`. The Neo4j schema version + moves 1.1.0 → 2.0.0, which forces a full re-upsert on the next incremental push. + +- **A failed Jelly leg is now reported at error level on JavaScript-majority + projects.** The union provider degrades to tsc-only when Jelly throws, and + reported that at `info`, which is not printed at default verbosity. On JavaScript + that is a ~81% edge loss with no signal (Jelly supplies 156 of 161 union edges on + NodeGoat). TypeScript projects keep the quieter `info` line. The default provider + is unchanged: `union` is a strict superset of `jelly` on JavaScript, measured both + with and without dependencies installed. +- **Caches from 0.5.0 and earlier are invalidated.** Extraction now produces more + callables from unchanged sources, so `ANALYZER_VERSION` moves with the release and + every cached `analysis_cache.json` is rebuilt on first run. + +### Measured on OWASP NodeGoat (dependencies installed, `-a 2`) + +| | 0.5.0 | 0.6.0 | +| --- | --- | --- | +| modules | 0 | 27 | +| callables | 0 | 59 | +| call-graph edges | 0 | 184 | +| resolved call sites | 0 | 51 | + +59 callables matches the parser-derived count of nameable functions in the source +exactly. The discovered module set equals the set of `.js` files outside +`node_modules`, `vendor` and test trees. + +### Known gaps +- CommonJS `require` / `module.exports` are not modelled at module level, so + `imports` and `exports` stay empty on CommonJS input. Relative `require()` **call + targets** do resolve. +- Method calls on an untyped receiver (e.g. `db.collection(...)` where `db` is an + untyped parameter) produce no edge into the library — tracked in #87. The call is + still attributable: it is recorded on the enclosing callable with its receiver + expression, and that callable is reachable from its route. diff --git a/package.json b/package.json index 6c13526..fd77859 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeanalyzer-typescript", - "version": "0.5.0", + "version": "0.6.0", "description": "CLDK TypeScript analyzer — emits the canonical CLDK analysis.json (symbol table + resolver-based call graph) via ts-morph.", "type": "module", "module": "src/index.ts", diff --git a/schema.neo4j.json b/schema.neo4j.json index c10af05..0716b4d 100644 --- a/schema.neo4j.json +++ b/schema.neo4j.json @@ -1,5 +1,5 @@ { - "schema_version": "1.1.0", + "schema_version": "2.0.0", "generator": "codeanalyzer-typescript", "marker_labels": [ "Entrypoint" @@ -267,7 +267,7 @@ ], "relationship_types": [ { - "type": "HAS_MODULE", + "type": "TS_HAS_MODULE", "from": [ "Application" ], @@ -277,7 +277,17 @@ "properties": {} }, { - "type": "DECLARES", + "type": "JS_HAS_MODULE", + "from": [ + "Application" + ], + "to": [ + "Module" + ], + "properties": {} + }, + { + "type": "TS_DECLARES", "from": [ "Module", "Namespace", @@ -295,7 +305,36 @@ "properties": {} }, { - "type": "HAS_METHOD", + "type": "JS_DECLARES", + "from": [ + "Module", + "Namespace", + "Class", + "Callable" + ], + "to": [ + "Class", + "Interface", + "Enum", + "TypeAlias", + "Namespace", + "Callable" + ], + "properties": {} + }, + { + "type": "TS_HAS_METHOD", + "from": [ + "Class", + "Interface" + ], + "to": [ + "Callable" + ], + "properties": {} + }, + { + "type": "JS_HAS_METHOD", "from": [ "Class", "Interface" @@ -306,7 +345,7 @@ "properties": {} }, { - "type": "HAS_ATTRIBUTE", + "type": "TS_HAS_ATTRIBUTE", "from": [ "Class", "Interface" @@ -317,7 +356,30 @@ "properties": {} }, { - "type": "DECLARES_VAR", + "type": "JS_HAS_ATTRIBUTE", + "from": [ + "Class", + "Interface" + ], + "to": [ + "Attribute" + ], + "properties": {} + }, + { + "type": "TS_DECLARES_VAR", + "from": [ + "Module", + "Namespace", + "Callable" + ], + "to": [ + "Variable" + ], + "properties": {} + }, + { + "type": "JS_DECLARES_VAR", "from": [ "Module", "Namespace", @@ -329,7 +391,17 @@ "properties": {} }, { - "type": "HAS_CALLSITE", + "type": "TS_HAS_CALLSITE", + "from": [ + "Callable" + ], + "to": [ + "CallSite" + ], + "properties": {} + }, + { + "type": "JS_HAS_CALLSITE", "from": [ "Callable" ], @@ -339,7 +411,18 @@ "properties": {} }, { - "type": "RESOLVES_TO", + "type": "TS_RESOLVES_TO", + "from": [ + "CallSite" + ], + "to": [ + "Callable", + "External" + ], + "properties": {} + }, + { + "type": "JS_RESOLVES_TO", "from": [ "CallSite" ], @@ -350,7 +433,7 @@ "properties": {} }, { - "type": "CALLS", + "type": "TS_CALLS", "from": [ "Callable" ], @@ -367,7 +450,36 @@ } }, { - "type": "EXTENDS", + "type": "JS_CALLS", + "from": [ + "Callable" + ], + "to": [ + "Callable", + "External" + ], + "properties": { + "weight": "integer", + "provenance": "string[]", + "dispatch": "string", + "external": "boolean", + "module": "string" + } + }, + { + "type": "TS_EXTENDS", + "from": [ + "Class", + "Interface" + ], + "to": [ + "Class", + "Interface" + ], + "properties": {} + }, + { + "type": "JS_EXTENDS", "from": [ "Class", "Interface" @@ -379,7 +491,17 @@ "properties": {} }, { - "type": "IMPLEMENTS", + "type": "TS_IMPLEMENTS", + "from": [ + "Class" + ], + "to": [ + "Interface" + ], + "properties": {} + }, + { + "type": "JS_IMPLEMENTS", "from": [ "Class" ], @@ -389,7 +511,22 @@ "properties": {} }, { - "type": "IMPORTS", + "type": "TS_IMPORTS", + "from": [ + "Module" + ], + "to": [ + "Module", + "Package" + ], + "properties": { + "imported_names": "string[]", + "import_kinds": "string[]", + "is_type_only": "boolean" + } + }, + { + "type": "JS_IMPORTS", "from": [ "Module" ], @@ -404,7 +541,18 @@ } }, { - "type": "RE_EXPORTS", + "type": "TS_RE_EXPORTS", + "from": [ + "Module" + ], + "to": [ + "Module", + "Package" + ], + "properties": {} + }, + { + "type": "JS_RE_EXPORTS", "from": [ "Module" ], @@ -415,7 +563,17 @@ "properties": {} }, { - "type": "MEMBER_OF", + "type": "TS_MEMBER_OF", + "from": [ + "External" + ], + "to": [ + "Package" + ], + "properties": {} + }, + { + "type": "JS_MEMBER_OF", "from": [ "External" ], @@ -425,7 +583,24 @@ "properties": {} }, { - "type": "DECORATED_BY", + "type": "TS_DECORATED_BY", + "from": [ + "Class", + "Callable", + "Attribute" + ], + "to": [ + "Decorator" + ], + "properties": { + "positional_arguments": "string[]", + "keyword_arguments_json": "string", + "start_line": "integer", + "end_line": "integer" + } + }, + { + "type": "JS_DECORATED_BY", "from": [ "Class", "Callable", @@ -458,21 +633,69 @@ "CREATE FULLTEXT INDEX code_fts IF NOT EXISTS FOR (c:Callable) ON EACH [c.code, c.docstring]" ], "label_twins": { - "Application": "TSApplication", - "Module": "TSModule", - "Class": "TSClass", - "Interface": "TSInterface", - "Enum": "TSEnum", - "TypeAlias": "TSTypeAlias", - "Namespace": "TSNamespace", - "Callable": "TSCallable", - "External": "TSExternal", - "AnonymousCallable": "TSAnonymousCallable", - "Package": "TSPackage", - "Decorator": "TSDecorator", - "CallSite": "TSCallSite", - "Attribute": "TSAttribute", - "Variable": "TSVariable", - "Entrypoint": "TSEntrypoint" + "Application": [ + "TSApplication", + "JSApplication" + ], + "Module": [ + "TSModule", + "JSModule" + ], + "Class": [ + "TSClass", + "JSClass" + ], + "Interface": [ + "TSInterface", + "JSInterface" + ], + "Enum": [ + "TSEnum", + "JSEnum" + ], + "TypeAlias": [ + "TSTypeAlias", + "JSTypeAlias" + ], + "Namespace": [ + "TSNamespace", + "JSNamespace" + ], + "Callable": [ + "TSCallable", + "JSCallable" + ], + "External": [ + "TSExternal", + "JSExternal" + ], + "AnonymousCallable": [ + "TSAnonymousCallable", + "JSAnonymousCallable" + ], + "Package": [ + "TSPackage", + "JSPackage" + ], + "Decorator": [ + "TSDecorator", + "JSDecorator" + ], + "CallSite": [ + "TSCallSite", + "JSCallSite" + ], + "Attribute": [ + "TSAttribute", + "JSAttribute" + ], + "Variable": [ + "TSVariable", + "JSVariable" + ], + "Entrypoint": [ + "TSEntrypoint", + "JSEntrypoint" + ] } } diff --git a/src/build/neo4j/bolt.ts b/src/build/neo4j/bolt.ts index c8c981c..ebe9bc5 100644 --- a/src/build/neo4j/bolt.ts +++ b/src/build/neo4j/bolt.ts @@ -21,7 +21,7 @@ import type { Logger } from "../../utils"; import type { EdgeRow, GraphRows, NodeRow, Prop } from "./rows"; import { chunk } from "./rows"; -import { CONSTRAINTS, INDEXES } from "./schema"; +import { CONSTRAINTS, INDEXES, nsAlt } from "./schema"; export interface BoltConfig { uri: string; @@ -30,7 +30,7 @@ export interface BoltConfig { database: string | null; } -const DESCENDANTS = "[:DECLARES|HAS_METHOD|HAS_ATTRIBUTE|DECLARES_VAR|HAS_CALLSITE*1..]"; +const DESCENDANTS = `[:${nsAlt("DECLARES", "HAS_METHOD", "HAS_ATTRIBUTE", "DECLARES_VAR", "HAS_CALLSITE")}*1..]`; const BATCH = 1000; export async function boltWriter( diff --git a/src/build/neo4j/cypher.ts b/src/build/neo4j/cypher.ts index aefe6f1..e0d0e89 100644 --- a/src/build/neo4j/cypher.ts +++ b/src/build/neo4j/cypher.ts @@ -9,7 +9,7 @@ import type { EdgeRow, GraphRows, NodeRow, Props } from "./rows"; import { chunk, cypherMap, cypherValue } from "./rows"; -import { CONSTRAINTS, INDEXES } from "./schema"; +import { CONSTRAINTS, INDEXES, nsAlt } from "./schema"; const BATCH = 500; @@ -37,8 +37,8 @@ function wipe(appName: string): string { const name = cypherValue(appName); return [ `MATCH (a:Application {name: ${name}})`, - "OPTIONAL MATCH (a)-[:HAS_MODULE]->(m:Module)", - "OPTIONAL MATCH (m)-[:DECLARES|HAS_METHOD|HAS_ATTRIBUTE|DECLARES_VAR|HAS_CALLSITE*1..]->(x)", + `OPTIONAL MATCH (a)-[:${nsAlt("HAS_MODULE")}]->(m:Module)`, + `OPTIONAL MATCH (m)-[:${nsAlt("DECLARES", "HAS_METHOD", "HAS_ATTRIBUTE", "DECLARES_VAR", "HAS_CALLSITE")}*1..]->(x)`, "DETACH DELETE x, m, a;", ].join("\n"); } diff --git a/src/build/neo4j/index.ts b/src/build/neo4j/index.ts index be460ec..fbff48b 100644 --- a/src/build/neo4j/index.ts +++ b/src/build/neo4j/index.ts @@ -3,6 +3,6 @@ export { project } from "./project"; export { renderCypher } from "./cypher"; export { boltWriter, type BoltConfig } from "./bolt"; -export { SCHEMA_VERSION, TS_PREFIX, twinOf, withTwins, buildSchemaDocument, NODE_LABELS, REL_TYPES, MARKER_LABELS } from "./schema"; +export { SCHEMA_VERSION, TS_PREFIX, twinOf, withTwins, buildSchemaDocument, NODE_LABELS, REL_TYPES, REL_TYPES_NS, MARKER_LABELS } from "./schema"; export type { SchemaDocument } from "./schema"; export type { GraphRows, NodeRow, EdgeRow } from "./rows"; diff --git a/src/build/neo4j/project.ts b/src/build/neo4j/project.ts index d112d3d..0ef8080 100644 Binary files a/src/build/neo4j/project.ts and b/src/build/neo4j/project.ts differ diff --git a/src/build/neo4j/rows.ts b/src/build/neo4j/rows.ts index 73d085a..d423dac 100644 Binary files a/src/build/neo4j/rows.ts and b/src/build/neo4j/rows.ts differ diff --git a/src/build/neo4j/schema.ts b/src/build/neo4j/schema.ts index 76a5ce5..dd5a2e7 100644 --- a/src/build/neo4j/schema.ts +++ b/src/build/neo4j/schema.ts @@ -13,7 +13,7 @@ * the :Application node of every emitted graph so any consumer can detect a producer/consumer * mismatch at runtime. */ -export const SCHEMA_VERSION = "1.1.0"; +export const SCHEMA_VERSION = "2.0.0"; export type PropType = "string" | "integer" | "float" | "boolean" | "string[]" | "integer[]"; @@ -43,20 +43,44 @@ export const MARKER_LABELS = ["Entrypoint"] as const; * constraints are unchanged. The bare labels drop (and rel types gain `TS_`) in schema 2.0.0. */ export const TS_PREFIX = "TS"; +export const JS_PREFIX = "JS"; -/** The TS-prefixed twin of a specific or marker label. */ -export const twinOf = (label: string): string => `${TS_PREFIX}${label}`; +/** Source language of a module path — the namespace its nodes and outgoing edges carry. */ +export const langOf = (fileKey: string): string => (/\.(js|jsx|mjs|cjs)$/.test(fileKey) ? JS_PREFIX : TS_PREFIX); + +/** + * The namespace a node belongs to. Nodes carrying `_module` take that module's language; the ones + * that have none of their own — the application root, packages, external library symbols — take + * the analyzer's own TS namespace, since a sibling analyzer emits its own (PY*, …). + */ +const nsOf = (props?: { _module?: unknown }): string => + typeof props?._module === "string" ? langOf(props._module) : TS_PREFIX; + +/** The namespaced twin of a specific or marker label. */ +export const twinOf = (label: string, ns: string = TS_PREFIX): string => `${ns}${label}`; + +/** A Cypher relationship-type alternation covering every namespace, e.g. `TS_DECLARES|JS_DECLARES`. */ +export const nsAlt = (...bases: string[]): string => + bases.flatMap((b) => [TS_PREFIX, JS_PREFIX].map((ns) => `${ns}_${b}`)).join("|"); + +/** An edge is namespaced by its source module's language, falling back to its target's. */ +export function relTypeFor(type: string, fromProps?: { _module?: unknown }, toProps?: { _module?: unknown }): string { + if (/^(TS|JS)_/.test(type)) return type; + const ns = typeof fromProps?._module === "string" ? langOf(fromProps._module) : nsOf(toProps); + return `${ns}_${type}`; +} /** * Expand a projection label set with its twins: order preserved, `Symbol` skipped, idempotent. * Any label already starting with `TS` is treated as a twin and never re-prefixed — so no bare * label may legitimately begin with `TS`. */ -export function withTwins(labels: string[]): string[] { +export function withTwins(labels: string[], props?: { _module?: unknown }): string[] { + const ns = nsOf(props); const out = [...labels]; for (const l of labels) { - if (l === "Symbol" || l.startsWith(TS_PREFIX)) continue; - const t = twinOf(l); + if (l === "Symbol" || l.startsWith(TS_PREFIX) || l.startsWith(JS_PREFIX)) continue; + const t = twinOf(l, ns); if (!out.includes(t)) out.push(t); } return out; @@ -375,15 +399,24 @@ export interface SchemaDocument { relationship_types: RelType[]; constraints: readonly string[]; indexes: readonly string[]; - /** Specific/marker label → its TS-prefixed twin (both are present on every emitted node). */ - label_twins: Record; + /** Specific/marker label → its namespaced twins, one per language. */ + label_twins: Record; } +/** + * Every relationship type the projection can emit: each declared type in both namespaces. REL_TYPES + * stays the single source of truth; this is derived so the two can never drift. + */ +export const REL_TYPES_NS: RelType[] = REL_TYPES.flatMap((r) => + [TS_PREFIX, JS_PREFIX].map((ns) => ({ ...r, type: `${ns}_${r.type}` })), +); + /** One twin per specific label + per marker label — derived from the catalogs, never drifts. */ -function labelTwins(): Record { - const out: Record = {}; - for (const n of NODE_LABELS) out[n.label] = twinOf(n.label); - for (const m of MARKER_LABELS) out[m] = twinOf(m); +function labelTwins(): Record { + const out: Record = {}; + const ns = [TS_PREFIX, JS_PREFIX]; + for (const n of NODE_LABELS) out[n.label] = ns.map((p) => twinOf(n.label, p)); + for (const m of MARKER_LABELS) out[m] = ns.map((p) => twinOf(m, p)); return out; } @@ -394,7 +427,7 @@ export function buildSchemaDocument(): SchemaDocument { generator: "codeanalyzer-typescript", marker_labels: MARKER_LABELS, node_labels: NODE_LABELS, - relationship_types: REL_TYPES, + relationship_types: REL_TYPES_NS, constraints: CONSTRAINTS, indexes: INDEXES, label_twins: labelTwins(), diff --git a/src/schema/signatures.ts b/src/schema/signatures.ts index cf91b74..52d65bd 100644 --- a/src/schema/signatures.ts +++ b/src/schema/signatures.ts @@ -4,7 +4,7 @@ * and the callee-side id (computed during call-graph resolution) are byte-identical. Edges can * therefore only ever reference signatures that exist in the symbol table. */ -import { Node } from "ts-morph"; +import { Node, SyntaxKind } from "ts-morph"; import { fileKeyOf, signatureOf, constructorSignatureOf } from "./schema"; /** The name a node contributes to a signature's dotted member chain, or null if it contributes none. */ @@ -19,6 +19,17 @@ export function contributorName(node: Node): string | null { if (Node.isGetAccessorDeclaration(node) || Node.isSetAccessorDeclaration(node)) return safeName(node); if (Node.isConstructorDeclaration(node)) return "constructor"; if (Node.isVariableDeclaration(node)) { + const init = node.getInitializer(); + if (init && (Node.isArrowFunction(init) || Node.isFunctionExpression(init))) return node.getName(); + // An object literal contributes its variable name so its methods are homed under it + // (`const api = { foo(){} }` → `.api.foo`). + if (init && Node.isObjectLiteralExpression(init)) return node.getName(); + return null; + } + // `this. = fn` inside a constructor function — the assignment is what names the callable. + if (Node.isBinaryExpression(node)) return thisAssignedFunctionName(node); + // `{ : function(){} }`. Shorthand `{ (){} }` is a MethodDeclaration, handled above. + if (Node.isPropertyAssignment(node)) { const init = node.getInitializer(); if (init && (Node.isArrowFunction(init) || Node.isFunctionExpression(init))) return node.getName(); return null; @@ -26,6 +37,22 @@ export function contributorName(node: Node): string | null { return null; } +/** + * The name in `this. = `, else null. Syntactic on purpose: `this` is lexical + * in an arrow and dynamic in a plain function, so a non-constructor still gets its members homed on + * it. Over-approximates rather than dropping the callable — deliberate, not a resolution. + */ +export function thisAssignedFunctionName(node: Node): string | null { + if (!Node.isBinaryExpression(node)) return null; + if (node.getOperatorToken().getKind() !== SyntaxKind.EqualsToken) return null; + const lhs = node.getLeft(); + if (!Node.isPropertyAccessExpression(lhs)) return null; + if (lhs.getExpression().getKind() !== SyntaxKind.ThisKeyword) return null; + const rhs = node.getRight(); + if (!Node.isArrowFunction(rhs) && !Node.isFunctionExpression(rhs)) return null; + return lhs.getName(); +} + export function isCallableDecl(node: Node): boolean { return ( Node.isFunctionDeclaration(node) || @@ -108,6 +135,14 @@ export function resolveCalleeSignature( return null; } + // `this. = fn` and `{ : function(){} }` are assignments, not declarations, so + // `isCallableDecl` does not cover them — but the checker hands them back as the declaration of + // the resolved property, and they are callables in the symbol table (issue #85). + if (Node.isBinaryExpression(decl) || Node.isPropertyAssignment(decl)) { + const s = computeSignatureForDecl(decl, root); + return s && allSignatures.has(s) ? { signature: s, isConstructor: false } : null; + } + if (isCallableDecl(decl)) { const s = computeSignatureForDecl(decl, root); return s && allSignatures.has(s) diff --git a/src/semantic_analysis/provider.ts b/src/semantic_analysis/provider.ts index f349a5f..724de84 100644 --- a/src/semantic_analysis/provider.ts +++ b/src/semantic_analysis/provider.ts @@ -77,6 +77,16 @@ function diffSummary(tsc: CallGraphResult, jelly: CallGraphResult): string { ); } +/** execFileSync puts the whole command — every entry file — in Error.message. Keep it readable. */ +const briefly = (reason: string): string => reason.split("\n", 1)[0]!.slice(0, 160); + +/** Whether most analyzed modules are JavaScript, i.e. whether the tsc leg has types to work with. */ +function isJavaScriptMajority(symbol_table: Record): boolean { + const files = Object.keys(symbol_table); + const js = files.filter((f) => /\.(js|jsx|mjs|cjs)$/.test(f)).length; + return files.length > 0 && js * 2 > files.length; +} + /** * Run tsc + jelly and emit their union. This is the default: jelly's edges and external symbols are * PERSISTED (tagged `provenance: ["jelly"]`) instead of being discarded after a diff. If jelly @@ -90,7 +100,17 @@ export const unionProvider: CallGraphProvider = { try { jelly = jellyProvider.build(ctx); } catch (e) { - ctx.log.info(`call graph (union): jelly failed (${(e as Error).message}); emitting tsc only`); + const reason = briefly((e as Error).message); + // Losing jelly is modest on TS, a cliff on JS (156 of 161 union edges on NodeGoat), and + // `info` is not printed at default verbosity — which made the JS case silent. + if (isJavaScriptMajority(ctx.symbol_table)) { + ctx.log.error( + `call graph (union): jelly failed (${reason}) on a JavaScript-majority project — ` + + `emitting tsc only, which typically loses most of the call graph`, + ); + } else { + ctx.log.info(`call graph (union): jelly failed (${reason}); emitting tsc only`); + } return tsc; } ctx.log.info(`call graph diff: ${diffSummary(tsc, jelly)}`); diff --git a/src/syntactic_analysis/builders.ts b/src/syntactic_analysis/builders.ts index f9a0fba..dc90950 100644 --- a/src/syntactic_analysis/builders.ts +++ b/src/syntactic_analysis/builders.ts @@ -26,7 +26,7 @@ import { constructorSignatureOf, fileKeyOf, } from "../schema"; -import { computeSignatureForDecl } from "../schema"; +import { computeSignatureForDecl, contributorName, thisAssignedFunctionName } from "../schema"; // ---------------------------------------------------------------------------------------------- // dynamic-getter helpers @@ -313,6 +313,31 @@ function namedBoundary(node: Node): Boundary { const init = node.getInitializer(); if (init && (Node.isArrowFunction(init) || Node.isFunctionExpression(init))) return "callable"; } + // `this. = fn` inside a constructor function (issue #85). + if (Node.isBinaryExpression(node) && thisAssignedFunctionName(node) !== null) return "callable"; + // Object-literal members: shorthand `{ foo(){} }` is a MethodDeclaration — one reachable from a + // function body can only be an object-literal member, since a class body is taken as "class" + // above and descent stops there. + if (Node.isMethodDeclaration(node)) return "callable"; + if (Node.isPropertyAssignment(node)) { + const init = node.getInitializer(); + if (init && (Node.isArrowFunction(init) || Node.isFunctionExpression(init))) return "callable"; + } + return null; +} + +/** The function node a "callable" boundary actually wraps, plus how to label it. */ +function callableOf(node: Node): { fnNode: Node; kind: TSCallableKind } | null { + if (Node.isFunctionDeclaration(node)) return { fnNode: node, kind: "function" }; + if (Node.isMethodDeclaration(node)) return { fnNode: node, kind: "method" }; + const init = Node.isVariableDeclaration(node) || Node.isPropertyAssignment(node) + ? node.getInitializer() + : Node.isBinaryExpression(node) + ? node.getRight() + : undefined; + if (!init) return null; + if (Node.isArrowFunction(init)) return { fnNode: init, kind: "arrow" }; + if (Node.isFunctionExpression(init)) return { fnNode: init, kind: "function_expression" }; return null; } @@ -411,16 +436,10 @@ export function buildCallable( onCall: (n) => call_sites.push(buildCallsite(n)), onLocal: (vd) => local_variables.push(buildVariable(vd, "function")), onNestedCallable: (n) => { - if (Node.isVariableDeclaration(n)) { - const init = n.getInitializer(); - if (!init) return; - const k: TSCallableKind = Node.isArrowFunction(init) ? "arrow" : "function_expression"; - const r = buildCallable(n, init, k, root); - if (r) inner_callables[r.sig] = r.callable; - } else { - const r = buildCallable(n, n, "function", root); - if (r) inner_callables[r.sig] = r.callable; - } + const c = callableOf(n); + if (!c) return; + const r = buildCallable(n, c.fnNode, c.kind, root); + if (r) inner_callables[r.sig] = r.callable; }, onNestedClass: (n) => { const r = buildClass(n, root); @@ -430,8 +449,11 @@ export function buildCallable( } const nameNode = sigNode as unknown as { getName?: () => string | undefined }; - const name = - Node.isConstructorDeclaration(fnNode) ? "constructor" : (nameNode.getName?.() ?? "(anonymous)"); + // An assignment-declared callable (`this.x = fn`) has no getName(), but the signature layer + // already knows what names it — fall back to that before giving up (issue #85). + const name = Node.isConstructorDeclaration(fnNode) + ? "constructor" + : (nameNode.getName?.() ?? contributorName(sigNode) ?? "(anonymous)"); const callable: TSCallable = { name, @@ -804,6 +826,16 @@ function buildStatemented(container: Node, root: string, varScope: TSVariableDec const r = buildCallable(vd, init, k, root); if (r) functions[r.sig] = r.callable; } else { + // A module-level object literal is still a variable, but its function-valued members are + // callables (issue #85) — `const api = { foo(){} }` is how much pre-class JS declares them. + if (init && Node.isObjectLiteralExpression(init)) { + for (const prop of init.getProperties()) { + const c = callableOf(prop); + if (!c) continue; + const r = buildCallable(prop, c.fnNode, c.kind, root); + if (r) functions[r.sig] = r.callable; + } + } variables.push(buildVariable(vd, varScope)); } } diff --git a/src/syntactic_analysis/discovery.ts b/src/syntactic_analysis/discovery.ts index fe7059b..20ded0e 100644 --- a/src/syntactic_analysis/discovery.ts +++ b/src/syntactic_analysis/discovery.ts @@ -2,7 +2,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { relPosix } from "../utils"; -const SOURCE_EXTS = new Set([".ts", ".tsx", ".mts", ".cts"]); +const SOURCE_EXTS = new Set([".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"]); const SKIP_DIRS = new Set([ "node_modules", @@ -23,7 +23,7 @@ const TEST_DIRS = new Set(["__tests__", "__test__", "test", "tests", "spec", "__ /** Test-ness is judged on the path RELATIVE TO the project root, never the absolute path. */ function isTestFile(relKey: string): boolean { const base = path.basename(relKey); - if (/\.(test|spec)\.(ts|tsx|mts|cts)$/.test(base)) return true; + if (/\.(test|spec)\.(ts|tsx|mts|cts|js|jsx|mjs|cjs)$/.test(base)) return true; return relKey.split("/").some((p) => TEST_DIRS.has(p)); } @@ -32,7 +32,12 @@ export interface DiscoveredFile { fileKey: string; // project-relative POSIX path with extension } -/** Recursively discover .ts/.tsx sources under root, skipping vendored and (optionally) test trees. */ +/** + * Recursively discover TypeScript and JavaScript sources under root, skipping vendored and + * (optionally) test trees. JavaScript is included because the checker already parses it — + * `defaultCompilerOptions()` sets `allowJs` — so the extension set was the only thing keeping + * plain-JS projects from being analyzed at all (issue #84). + */ export function discoverSourceFiles(root: string, skipTests: boolean): DiscoveredFile[] { const out: DiscoveredFile[] = []; const walk = (dir: string): void => { diff --git a/src/syntactic_analysis/symbolTable.ts b/src/syntactic_analysis/symbolTable.ts index 62f16a6..22b41b9 100644 --- a/src/syntactic_analysis/symbolTable.ts +++ b/src/syntactic_analysis/symbolTable.ts @@ -27,6 +27,9 @@ export function buildSymbolTable( const targets = opts.targetFiles ? resolveTargetFiles(root, opts.targetFiles) : null; const allProjectFiles = discoverSourceFiles(root, opts.skipTests); + if (allProjectFiles.length === 0) { + log.warn(`no source files found under ${root} — nothing to analyze`); + } // The set of files to BUILD (targets in -t mode, else all). const buildFiles = targets ?? allProjectFiles; // Add ALL project files to the program so cross-file resolution works even in -t mode. diff --git a/src/utils/version.ts b/src/utils/version.ts index c33aa41..a97f704 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -3,4 +3,4 @@ * the analyzer invalidates stale per-file Modules (whose source is unchanged but whose extracted * shape may differ across analyzer versions). */ -export const ANALYZER_VERSION = "0.5.0"; +export const ANALYZER_VERSION = "0.6.0"; diff --git a/test/fixtures/idiom-app/package.json b/test/fixtures/idiom-app/package.json new file mode 100644 index 0000000..3b3b4ef --- /dev/null +++ b/test/fixtures/idiom-app/package.json @@ -0,0 +1 @@ +{ "name": "idiom-app", "version": "1.0.0", "private": true } diff --git a/test/fixtures/idiom-app/src/caller.js b/test/fixtures/idiom-app/src/caller.js new file mode 100644 index 0000000..3d70cdc --- /dev/null +++ b/test/fixtures/idiom-app/src/caller.js @@ -0,0 +1,9 @@ +const { Dao } = require("./ctorfn"); +const api = require("./objlit"); + +function run(db, id) { + const dao = new Dao(db); + return dao.getById(id) + api.getById(id); +} + +module.exports = { run }; diff --git a/test/fixtures/idiom-app/src/ctorfn.js b/test/fixtures/idiom-app/src/ctorfn.js new file mode 100644 index 0000000..d4c3071 --- /dev/null +++ b/test/fixtures/idiom-app/src/ctorfn.js @@ -0,0 +1,13 @@ +function Dao(db) { + const helper = () => "h"; + + this.getById = (id) => { + return helper() + db.collection("x").find(id); + }; + + this.save = function (row) { + return db.collection("x").insert(row); + }; +} + +module.exports = { Dao }; diff --git a/test/fixtures/idiom-app/src/ctorfn_ts.ts b/test/fixtures/idiom-app/src/ctorfn_ts.ts new file mode 100644 index 0000000..33cebef --- /dev/null +++ b/test/fixtures/idiom-app/src/ctorfn_ts.ts @@ -0,0 +1,7 @@ +function Dao2(this: any, db: any) { + this.getById = (id: number) => { + return db.collection("x").find(id); + }; +} + +export { Dao2 }; diff --git a/test/fixtures/idiom-app/src/objlit.js b/test/fixtures/idiom-app/src/objlit.js new file mode 100644 index 0000000..79a6517 --- /dev/null +++ b/test/fixtures/idiom-app/src/objlit.js @@ -0,0 +1,10 @@ +const api = { + getById(id) { + return id; + }, + save: function (row) { + return row; + }, +}; + +module.exports = api; diff --git a/test/fixtures/idiom-app/src/objlit_ts.ts b/test/fixtures/idiom-app/src/objlit_ts.ts new file mode 100644 index 0000000..8d5a96d --- /dev/null +++ b/test/fixtures/idiom-app/src/objlit_ts.ts @@ -0,0 +1,7 @@ +const api2 = { + getById(id: number) { + return id; + }, +}; + +export { api2 }; diff --git a/test/fixtures/js-app/README.md b/test/fixtures/js-app/README.md new file mode 100644 index 0000000..84985c9 --- /dev/null +++ b/test/fixtures/js-app/README.md @@ -0,0 +1 @@ +Fixture: a plain CommonJS/ESM JavaScript app with no tsconfig. diff --git a/test/fixtures/js-app/package.json b/test/fixtures/js-app/package.json new file mode 100644 index 0000000..3984e34 --- /dev/null +++ b/test/fixtures/js-app/package.json @@ -0,0 +1,6 @@ +{ + "name": "js-app", + "version": "1.0.0", + "private": true, + "main": "src/index.js" +} diff --git a/test/fixtures/js-app/src/helpers.mjs b/test/fixtures/js-app/src/helpers.mjs new file mode 100644 index 0000000..02b9ab8 --- /dev/null +++ b/test/fixtures/js-app/src/helpers.mjs @@ -0,0 +1,3 @@ +export function titleCase(value) { + return value.charAt(0).toUpperCase() + value.slice(1); +} diff --git a/test/fixtures/js-app/src/index.js b/test/fixtures/js-app/src/index.js new file mode 100644 index 0000000..4c9a189 --- /dev/null +++ b/test/fixtures/js-app/src/index.js @@ -0,0 +1,7 @@ +const { slugify, truncate } = require("./util"); + +function makeHandle(name, limit) { + return truncate(slugify(name), limit); +} + +module.exports = makeHandle; diff --git a/test/fixtures/js-app/src/legacy.cjs b/test/fixtures/js-app/src/legacy.cjs new file mode 100644 index 0000000..e7a2c26 --- /dev/null +++ b/test/fixtures/js-app/src/legacy.cjs @@ -0,0 +1,3 @@ +exports.pad = function pad(value, width) { + return String(value).padStart(width, "0"); +}; diff --git a/test/fixtures/js-app/src/util.js b/test/fixtures/js-app/src/util.js new file mode 100644 index 0000000..d579d64 --- /dev/null +++ b/test/fixtures/js-app/src/util.js @@ -0,0 +1,9 @@ +function slugify(value) { + return String(value).trim().toLowerCase().replace(/\s+/g, "-"); +} + +function truncate(value, limit) { + return value.length > limit ? value.slice(0, limit) : value; +} + +module.exports = { slugify, truncate }; diff --git a/test/fixtures/js-app/src/util.test.js b/test/fixtures/js-app/src/util.test.js new file mode 100644 index 0000000..7cb58a7 --- /dev/null +++ b/test/fixtures/js-app/src/util.test.js @@ -0,0 +1,10 @@ +// Fixture data, not a real test: named `.test.js` only so discovery's skip-tests classifier +// has something to classify. Deliberately contains no test-runner calls so that `bun test` +// does not execute fixture files as part of the analyzer's own suite. +const { slugify } = require("./util"); + +function expectedHandle(name) { + return slugify(name); +} + +module.exports = { expectedHandle }; diff --git a/test/fixtures/js-app/src/widget.jsx b/test/fixtures/js-app/src/widget.jsx new file mode 100644 index 0000000..cc8e91c --- /dev/null +++ b/test/fixtures/js-app/src/widget.jsx @@ -0,0 +1,3 @@ +export function Widget(props) { + return
{props.label}
; +} diff --git a/test/idiom-callables.test.ts b/test/idiom-callables.test.ts new file mode 100644 index 0000000..df159a0 --- /dev/null +++ b/test/idiom-callables.test.ts @@ -0,0 +1,139 @@ +/** + * Issue #85: two ways of declaring a method were never materialized as callables, so calls to + * them could not resolve — edges are gated to `allSignatures`, which is built from the symbol + * table. Both are missed identically in TypeScript, so this is a language-neutral gap: + * + * • `this. = fn` inside a constructor function — landed in `local_variables` + * • object-literal methods (`{ foo(){} }`, `{ foo: function(){} }`) — not emitted at all + * + * On OWASP NodeGoat this left 24 callables against 115 function-like nodes in source, and the + * whole DAO method layer absent from the call graph. + */ +import { describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { analyze } from "../src/core"; +import type { AnalysisOptions } from "../src/options"; +import type { TSApplication, TSCallable, TSClass, TSModule } from "../src/schema"; + +const FIXTURE = path.resolve(import.meta.dir, "fixtures/idiom-app"); + +function analyzeFixture(): TSApplication { + const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), "cants-idiom-")); + const opts: AnalysisOptions = { + input: FIXTURE, output: null, emit: "json", appName: "idiom-app", + neo4jUri: null, neo4jUser: "neo4j", neo4jPassword: "", neo4jDatabase: null, + analysisLevel: 2, targetFiles: null, skipTests: true, eager: true, + noBuild: true, phantoms: true, callGraphProvider: "tsc", cacheDir, verbosity: 0, + }; + try { + return analyze(opts); + } finally { + fs.rmSync(cacheDir, { recursive: true, force: true }); + } +} + +/** Every callable signature in the symbol table, including nested ones. */ +function callableSignatures(app: TSApplication): Set { + const out = new Set(); + const walkCallable = (c: TSCallable): void => { + out.add(c.signature); + for (const inner of Object.values(c.inner_callables ?? {})) walkCallable(inner); + for (const cls of Object.values(c.inner_classes ?? {})) walkClass(cls); + }; + const walkClass = (k: TSClass): void => { + for (const m of Object.values(k.methods ?? {})) walkCallable(m); + for (const inner of Object.values(k.inner_classes ?? {})) walkClass(inner); + }; + for (const m of Object.values(app.symbol_table) as TSModule[]) { + for (const c of Object.values(m.functions ?? {})) walkCallable(c); + for (const k of Object.values(m.classes ?? {})) walkClass(k); + } + return out; +} + +describe("callables declared through dynamic idioms", () => { + const app = analyzeFixture(); + const sigs = callableSignatures(app); + + test("materializes `this. = fn` inside a constructor function", () => { + expect(sigs).toContain("src/ctorfn.Dao.getById"); + expect(sigs).toContain("src/ctorfn.Dao.save"); + }); + + test("materializes the same idiom in TypeScript", () => { + expect(sigs).toContain("src/ctorfn_ts.Dao2.getById"); + }); + + test("materializes object-literal methods", () => { + expect(sigs).toContain("src/objlit.api.getById"); + expect(sigs).toContain("src/objlit.api.save"); + }); + + test("materializes object-literal methods in TypeScript", () => { + expect(sigs).toContain("src/objlit_ts.api2.getById"); + }); + + test("still materializes plain nested callables (no regression)", () => { + expect(sigs).toContain("src/ctorfn.Dao.helper"); + }); +}); + +describe("edges into callables declared through dynamic idioms", () => { + const app = analyzeFixture(); + const targets = app.call_graph.filter((e) => e.source === "src/caller.run").map((e) => e.target).sort(); + + test("a call through a constructor-function instance resolves", () => { + expect(targets).toContain("src/ctorfn.Dao.getById"); + }); + + test("a call on an object literal resolves", () => { + expect(targets).toContain("src/objlit.api.getById"); + }); +}); + +describe("names of callables declared through dynamic idioms", () => { + const app = analyzeFixture(); + const named = new Map(); + const walk = (c: TSCallable): void => { + named.set(c.signature, c.name); + for (const inner of Object.values(c.inner_callables ?? {})) walk(inner); + }; + for (const m of Object.values(app.symbol_table) as TSModule[]) { + for (const c of Object.values(m.functions ?? {})) walk(c); + } + + test("a `this. = fn` callable is named, not (anonymous)", () => { + expect(named.get("src/ctorfn.Dao.getById")).toBe("getById"); + expect(named.get("src/ctorfn.Dao.save")).toBe("save"); + }); + + test("an object-literal member is named", () => { + expect(named.get("src/objlit.api.save")).toBe("save"); + }); +}); + +describe("security-scoped attribution: a sink call is homed on a named, reachable callable", () => { + const app = analyzeFixture(); + const dao = Object.values(app.symbol_table as Record) + .flatMap((m) => Object.values(m.functions ?? {})) + .flatMap((f) => Object.values(f.inner_callables ?? {})) + .find((c) => c.signature === "src/ctorfn.Dao.getById"); + + test("the DAO method exists and carries its sink call site", () => { + expect(dao).toBeDefined(); + const sinks = (dao?.call_sites ?? []).filter((s) => s.method_name === "find"); + expect(sinks.length).toBeGreaterThan(0); + // Attribution without a fabricated edge: the receiver expression is recorded, so a consumer + // can chain it to the enclosing callable's parameters/locals. + expect(sinks[0]?.receiver_expr).toBeTruthy(); + }); + + test("the enclosing constructor records the parameter the receiver derives from", () => { + const ctor = Object.values(app.symbol_table as Record) + .flatMap((m) => Object.values(m.functions ?? {})) + .find((c) => c.signature === "src/ctorfn.Dao"); + expect(ctor?.parameters.map((p) => p.name)).toContain("db"); + }); +}); diff --git a/test/jelly-degradation.test.ts b/test/jelly-degradation.test.ts new file mode 100644 index 0000000..61da3e8 --- /dev/null +++ b/test/jelly-degradation.test.ts @@ -0,0 +1,84 @@ +/** + * Issue #84: jelly supplies the large majority of a JavaScript project's call graph — on OWASP + * NodeGoat with dependencies installed, 156 of 161 union edges. When the jelly leg fails, the + * union provider degrades to tsc only, a ~81% edge loss on JS, reported at `info` level: not + * printed at all at default verbosity. That silent cliff must be loud. + */ +import { describe, expect, spyOn, test } from "bun:test"; +import { Project } from "ts-morph"; +import type { CallGraphContext, CallGraphResult } from "../src/semantic_analysis"; +import { jellyProvider, tscProvider, unionProvider } from "../src/semantic_analysis"; +import type { TSModule } from "../src/schema"; +import { Logger } from "../src/utils/logging"; + +class RecordingLogger extends Logger { + readonly infos: string[] = []; + readonly errors: string[] = []; + override info(msg: string): void { + this.infos.push(msg); + } + override warn(msg: string): void { + this.errors.push(msg); + } + override error(msg: string): void { + this.errors.push(msg); + } +} + +const EMPTY: CallGraphResult = { edges: [], external_symbols: {}, synthesized_callables: {} }; + +function contextOver(files: string[], log: Logger): CallGraphContext { + const symbol_table: Record = {}; + for (const f of files) symbol_table[f] = {} as TSModule; + return { + project: new Project({ useInMemoryFileSystem: true }), + symbol_table, + root: "/tmp/project", + log, + phantoms: true, + }; +} + +/** Run the union provider with a failing jelly leg and a stubbed tsc leg. */ +function unionWithFailingJelly(files: string[], reason = "jelly exited 1"): RecordingLogger { + const log = new RecordingLogger(0); + const tsc = spyOn(tscProvider, "build").mockImplementation(() => EMPTY); + const jelly = spyOn(jellyProvider, "build").mockImplementation(() => { + throw new Error(reason); + }); + try { + unionProvider.build(contextOver(files, log)); + } finally { + tsc.mockRestore(); + jelly.mockRestore(); + } + return log; +} + +describe("union provider when the jelly leg fails", () => { + test("escalates on a JavaScript-majority project", () => { + const log = unionWithFailingJelly(["src/a.js", "src/b.js", "src/c.cjs", "src/d.ts"]); + + expect(log.errors.join("\n")).toContain("jelly"); + expect(log.errors.join("\n")).toContain("JavaScript"); + }); + + test("does not inline execFileSync's whole command line into the message", () => { + // execFileSync sets `Command failed: node …/jelly.js `, which on NodeGoat is + // 27 paths — unreadable in an error the user is meant to act on. + const reason = `Command failed: node /x/jelly.js -j /tmp/out.json ${"app/routes/thing.js ".repeat(60)}`; + + const log = unionWithFailingJelly(["src/a.js", "src/b.js", "src/c.js"], reason); + + const msg = log.errors.join("\n"); + expect(msg).toContain("Command failed"); + expect(msg.length).toBeLessThan(300); + }); + + test("stays at info level on a TypeScript-majority project", () => { + const log = unionWithFailingJelly(["src/a.ts", "src/b.ts", "src/c.tsx", "src/d.js"]); + + expect(log.errors).toEqual([]); + expect(log.infos.join("\n")).toContain("jelly failed"); + }); +}); diff --git a/test/js-discovery.test.ts b/test/js-discovery.test.ts new file mode 100644 index 0000000..e4ca951 --- /dev/null +++ b/test/js-discovery.test.ts @@ -0,0 +1,127 @@ +/** + * Discovery must see JavaScript, not just TypeScript (issue #84). Before this, `SOURCE_EXTS` + * held only the four TS extensions, so a JS-only project produced an empty symbol table and + * `cants` exited 0 with no warning. + */ +import { describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { AnalysisOptions } from "../src/options"; +import { analyze } from "../src/core"; +import { discoverSourceFiles } from "../src/syntactic_analysis/discovery"; +import { buildSymbolTable } from "../src/syntactic_analysis/symbolTable"; +import { Logger } from "../src/utils/logging"; + +const JS_APP = path.resolve(import.meta.dir, "fixtures/js-app"); + +const keysOf = (skipTests: boolean): string[] => discoverSourceFiles(JS_APP, skipTests).map((f) => f.fileKey); + +describe("discoverSourceFiles on a JavaScript project", () => { + test("discovers .js, .jsx, .mjs and .cjs sources", () => { + expect(keysOf(false)).toEqual([ + "src/helpers.mjs", + "src/index.js", + "src/legacy.cjs", + "src/util.js", + "src/util.test.js", + "src/widget.jsx", + ]); + }); + + test("treats .test.js as a test file when skipTests is on", () => { + expect(keysOf(true)).toEqual([ + "src/helpers.mjs", + "src/index.js", + "src/legacy.cjs", + "src/util.js", + "src/widget.jsx", + ]); + }); +}); + +const optionsFor = (input: string): AnalysisOptions => ({ + input, + output: null, + emit: "json", + appName: null, + neo4jUri: null, + neo4jUser: "neo4j", + neo4jPassword: "neo4j", + neo4jDatabase: null, + analysisLevel: 1, + targetFiles: null, + skipTests: true, + eager: false, + noBuild: true, + phantoms: true, + callGraphProvider: "union", + cacheDir: null, + verbosity: 0, +}); + +describe("analyze() on a JavaScript project", () => { + const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), "cants-js-app-")); + const app = (() => { + try { + return analyze({ ...optionsFor(JS_APP), appName: "js-app", eager: true, cacheDir }); + } finally { + fs.rmSync(cacheDir, { recursive: true, force: true }); + } + })(); + + test("builds a module for every discovered JavaScript source", () => { + expect(Object.keys(app.symbol_table).sort()).toEqual([ + "src/helpers.mjs", + "src/index.js", + "src/legacy.cjs", + "src/util.js", + "src/widget.jsx", + ]); + }); + + test("resolves calls across a relative require()", () => { + // index.js calls both helpers it destructured off `require("./util")`. Signatures carry no + // file extension — `stripTsExtension` (src/schema/schema.ts) strips .js/.jsx/.mjs/.cjs too. + const edges = app.call_graph.filter((e) => e.source === "src/index.makeHandle"); + + expect(edges.map((e) => e.target).sort()).toEqual(["src/util.slugify", "src/util.truncate"]); + }); + + test("both call-graph providers see the JavaScript sources", () => { + const provenance = new Set(app.call_graph.flatMap((e) => e.provenance)); + + expect(provenance.has("tsc")).toBe(true); + expect(provenance.has("jelly")).toBe(true); + }); +}); + +/** A real Logger that records warnings instead of writing them, so the test can assert on them. */ +class RecordingLogger extends Logger { + readonly warnings: string[] = []; + override warn(msg: string): void { + this.warnings.push(msg); + } +} + +describe("buildSymbolTable on a project with no analyzable sources", () => { + test("warns instead of succeeding silently", () => { + const empty = fs.mkdtempSync(path.join(os.tmpdir(), "cants-empty-")); + fs.writeFileSync(path.join(empty, "README.md"), "no sources here\n"); + const log = new RecordingLogger(0); + + try { + const result = buildSymbolTable( + optionsFor(empty), + { tsConfigFilePath: null, degraded: false, notes: [] }, + null, + log, + ); + expect(Object.keys(result.symbol_table)).toEqual([]); + } finally { + fs.rmSync(empty, { recursive: true, force: true }); + } + + expect(log.warnings.join("\n")).toContain("no source files"); + }); +}); diff --git a/test/neo4j-bolt.test.ts b/test/neo4j-bolt.test.ts index 65c8eca..774e9ab 100644 --- a/test/neo4j-bolt.test.ts +++ b/test/neo4j-bolt.test.ts @@ -108,7 +108,7 @@ containerSuite("neo4j bolt writer", () => { // A known resolved call edge from the fixture (index.ts calls services.announce). expect( await num( - "MATCH (:Callable)-[:CALLS]->(t:Callable {name:$n}) RETURN count(*)", + "MATCH (:Callable)-[:TS_CALLS|JS_CALLS]->(t:Callable {name:$n}) RETURN count(*)", { n: "announce" }, ), ).toBeGreaterThan(0); diff --git a/test/neo4j-lang-prefix.test.ts b/test/neo4j-lang-prefix.test.ts new file mode 100644 index 0000000..17d5f9f --- /dev/null +++ b/test/neo4j-lang-prefix.test.ts @@ -0,0 +1,56 @@ +/** + * Node labels and relationship types are namespaced per source language: TS for TypeScript, JS for + * JavaScript. Without this, a database holding more than one analyzer's output mingles them — + * codeanalyzer-python already namespaces every edge (PY_CALLS, PY_DECLARES, …) while this analyzer + * emitted bare CALLS/DECLARES. + * + * Nodes with no language of their own — the application root, npm packages, external library + * symbols — carry the analyzer's own TS namespace, since a sibling analyzer emits its own. + */ +import { describe, expect, test } from "bun:test"; +import { project } from "../src/build/neo4j"; +import { CALL_DEP, type TSApplication, type TSCallable, type TSModule } from "../src/schema"; + +const callable = (signature: string, name: string, path: string): TSCallable => + ({ signature, name, path }) as unknown as TSCallable; + +const mod = (fns: Record): TSModule => + ({ functions: fns, classes: {}, interfaces: {}, enums: {}, type_aliases: {}, namespaces: {}, variables: [], imports: [], exports: [], comments: [] }) as unknown as TSModule; + +const app: TSApplication = { + symbol_table: { + "src/a.js": mod({ aj: callable("src/a.aj", "aj", "/p/src/a.js") }), + "src/b.ts": mod({ bt: callable("src/b.bt", "bt", "/p/src/b.ts") }), + }, + call_graph: [{ source: "src/a.aj", target: "src/b.bt", type: CALL_DEP, weight: 1, provenance: ["tsc"], tags: {} }], + external_symbols: {}, + synthesized_callables: {}, +} as unknown as TSApplication; + +const rows = project(app, "mixed"); +const labelsOf = (value: string): string[] => rows.nodes.find((n) => n.value === value)?.labels ?? []; + +describe("per-language namespacing in the neo4j projection", () => { + test("a JavaScript module is labelled JSModule, not TSModule", () => { + expect(labelsOf("src/a.js")).toContain("JSModule"); + expect(labelsOf("src/a.js")).not.toContain("TSModule"); + }); + + test("a TypeScript module is labelled TSModule", () => { + expect(labelsOf("src/b.ts")).toContain("TSModule"); + }); + + test("every relationship type is namespaced", () => { + const bare = rows.edges.filter((e) => !/^(TS|JS)_/.test(e.type)).map((e) => e.type); + expect(bare).toEqual([]); + }); + + test("an edge takes its source module's language", () => { + const call = rows.edges.find((e) => e.type.endsWith("_CALLS")); + expect(call?.type).toBe("JS_CALLS"); + }); + + test("the application root keeps the analyzer's own TS namespace", () => { + expect(labelsOf("mixed")).toContain("TSApplication"); + }); +}); diff --git a/test/neo4j-schema.test.ts b/test/neo4j-schema.test.ts index 8f56659..b89a51a 100644 --- a/test/neo4j-schema.test.ts +++ b/test/neo4j-schema.test.ts @@ -12,7 +12,7 @@ import * as path from "node:path"; import { MARKER_LABELS, NODE_LABELS, - REL_TYPES, + REL_TYPES_NS, buildSchemaDocument, project, twinOf, @@ -40,7 +40,7 @@ function fixtureRows() { const byLabel = new Map(NODE_LABELS.map((n) => [n.label, n])); const mergeOf = new Map(NODE_LABELS.map((n) => [n.label, n.mergeLabel])); -const relByType = new Map(REL_TYPES.map((r) => [r.type, r])); +const relByType = new Map(REL_TYPES_NS.map((r) => [r.type, r])); const markers = new Set(MARKER_LABELS); const twins = new Set([ ...NODE_LABELS.map((n) => twinOf(n.label)), @@ -93,7 +93,7 @@ describe("neo4j schema conformance", () => { expect(onDisk).toBe(fresh); }); - test("every node carries exactly the TS twins of its base labels (1.1.0 dual-labeling)", () => { + test("every node carries exactly the TS twins of its base labels (2.0.0 dual-labeling)", () => { for (const node of rows.nodes) { const base = node.labels.filter((l) => !twins.has(l)); expect(new Set(node.labels), `bad twin set on ${node.labels.join(":")} ${node.value}`).toEqual( @@ -103,9 +103,9 @@ describe("neo4j schema conformance", () => { } }); - test(":Application is stamped with the 1.1.0 contract version", () => { + test(":Application is stamped with the 2.0.0 contract version", () => { const app = rows.nodes.find((n) => n.labels[0] === "Application"); expect(app).toBeDefined(); - expect(app!.props.schema_version).toBe("1.1.0"); + expect(app!.props.schema_version).toBe("2.0.0"); }); }); diff --git a/test/neo4j-twins.test.ts b/test/neo4j-twins.test.ts index 62db1a5..77f7fe4 100644 --- a/test/neo4j-twins.test.ts +++ b/test/neo4j-twins.test.ts @@ -1,5 +1,5 @@ /** - * Twin-label vocabulary (graph schema 1.1.0, issue #65): every specific and marker label has a + * Twin-label vocabulary (graph schema 2.0.0, issue #65): every specific and marker label has a * TS-prefixed twin; the shared merge label `Symbol` deliberately has none (epic #64). */ import { describe, expect, test } from "bun:test"; @@ -30,14 +30,14 @@ describe("TS twin-label vocabulary", () => { expect(withTwins(["Module", "TSModule"])).toEqual(["Module", "TSModule"]); }); - test("schema version is 1.1.0 (additive MINOR)", () => { - expect(SCHEMA_VERSION).toBe("1.1.0"); + test("schema version is 2.0.0 (additive MINOR)", () => { + expect(SCHEMA_VERSION).toBe("2.0.0"); }); test("schema document maps every specific + marker label to its twin", () => { const doc = buildSchemaDocument(); - for (const n of NODE_LABELS) expect(doc.label_twins[n.label]).toBe(twinOf(n.label)); - for (const m of MARKER_LABELS) expect(doc.label_twins[m]).toBe(twinOf(m)); + for (const n of NODE_LABELS) expect(doc.label_twins[n.label]).toEqual([twinOf(n.label, "TS"), twinOf(n.label, "JS")]); + for (const m of MARKER_LABELS) expect(doc.label_twins[m]).toEqual([twinOf(m, "TS"), twinOf(m, "JS")]); expect(doc.label_twins["Symbol"]).toBeUndefined(); expect(Object.keys(doc.label_twins).length).toBe(NODE_LABELS.length + MARKER_LABELS.length); }); diff --git a/test/synthesized-nodes.test.ts b/test/synthesized-nodes.test.ts index 21fe613..e7a8769 100644 --- a/test/synthesized-nodes.test.ts +++ b/test/synthesized-nodes.test.ts @@ -28,12 +28,12 @@ describe("synthesized anonymous-callable nodes", () => { }); test("the CALLS edge to the anonymous callable survives (was silently dropped before)", () => { - const e = rows.edges.find((e) => e.type === "CALLS" && e.to.value === ANON); + const e = rows.edges.find((e) => e.type === "TS_CALLS" && e.to.value === ANON); expect(e?.from.value).toBe("src/x.foo"); }); test("a DECLARES edge links the host symbol to it (keeps it in the wiped subgraph)", () => { - const e = rows.edges.find((e) => e.type === "DECLARES" && e.to.value === ANON); + const e = rows.edges.find((e) => e.type === "TS_DECLARES" && e.to.value === ANON); expect(e?.from.value).toBe("src/x.foo"); expect(e?.from.label).toBe("Symbol"); });