From 4101a6e66dc06e8081c5f8236872ac4909d179f0 Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Thu, 17 Sep 2026 14:00:49 +0000 Subject: [PATCH 1/4] chore: migrate away from deprecated vm2 to node:vm --- cli/vm/BUILD | 3 +- cli/vm/compile.ts | 40 +-- cli/vm/jit_worker.ts | 22 +- common/vm/BUILD | 50 ++++ common/vm/vm_runner.ts | 412 ++++++++++++++++++++++++++++++ common/vm/vm_runner_benchmark.ts | 46 ++++ common/vm/vm_runner_test.ts | 372 +++++++++++++++++++++++++++ core/main_property_graphs_test.ts | 14 +- package.json | 1 - packages/@dataform/cli/BUILD | 1 - packages/rollup.config.js | 4 +- testing/BUILD | 2 +- testing/run_core.ts | 33 +-- yarn.lock | 15 -- 14 files changed, 935 insertions(+), 80 deletions(-) create mode 100644 common/vm/BUILD create mode 100644 common/vm/vm_runner.ts create mode 100644 common/vm/vm_runner_benchmark.ts create mode 100644 common/vm/vm_runner_test.ts diff --git a/cli/vm/BUILD b/cli/vm/BUILD index 4adc5f391..a8fc54948 100644 --- a/cli/vm/BUILD +++ b/cli/vm/BUILD @@ -1,5 +1,6 @@ package(default_visibility = ["//visibility:public"]) +load("//testing:index.bzl", "ts_test_suite") load("//tools:ts_library.bzl", "ts_library") ts_library( @@ -10,6 +11,7 @@ ts_library( ], deps = [ "//common/protos", + "//common/vm:vm_runner", "//core", "//protos:ts", "@npm//@types/glob", @@ -17,7 +19,6 @@ ts_library( "@npm//@types/semver", "@npm//glob", "@npm//semver", - "@npm//vm2", ], ) diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index 6626fbac1..7a0c1d911 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -2,9 +2,9 @@ import * as fs from "fs"; import * as glob from "glob"; import * as path from "path"; import * as semver from "semver"; -import { CompilerFunction, NodeVM } from "vm2"; import { encode64 } from "df/common/protos"; +import { CompilerFunction, VmRunner } from "df/common/vm/vm_runner"; import { dataform } from "df/protos/ts"; export function compile(compileConfig: dataform.ICompileConfig) { @@ -31,14 +31,9 @@ export function compile(compileConfig: dataform.ICompileConfig) { // through Node's resolver inside the vm covers every install layout // (package.json, workflow_settings.yaml, JiT) and matches what the user's // code will see. require() caches the bundle so the second call is free. - const indexGeneratorVm = new NodeVM({ - wrapper: "none", - require: { - context: "sandbox", - root: compileConfig.projectDir, - external: true, - builtin: ["path"], - }, + const indexGeneratorVm = new VmRunner({ + projectDir: compileConfig.projectDir, + builtinModules: ["path"], }); const compiler: CompilerFunction = indexGeneratorVm.run( 'return require("@dataform/core").compiler', @@ -71,15 +66,15 @@ export function compile(compileConfig: dataform.ICompileConfig) { } const needsCallerFileShim = semver.lt(dataformCoreVersion, "3.0.57"); - // vm2 strips file paths from V8 CallSite objects inside the sandbox, so - // getCallerFile() in @dataform/core needs a fallback. Track the currently + // While VmRunner preserves V8 CallSite file paths natively, older @dataform/core + // versions check global.__dataform_current_file as a fallback. Track the currently // executing file via a host-side stack exposed through sandbox helpers, and // expose it as a getter on `global.__dataform_current_file`. const fileStack: string[] = []; - // Then use vm2's native compiler integration to apply the compiler to files. - const userCodeVm = new NodeVM({ - wrapper: "none", + // Then use VmRunner to apply the compiler to files. + const userCodeVm = new VmRunner({ + projectDir: compileConfig.projectDir, sandbox: { __df_enter: (p: string) => { fileStack.push(p); @@ -89,19 +84,10 @@ export function compile(compileConfig: dataform.ICompileConfig) { }, __df_current: () => (fileStack.length > 0 ? fileStack[fileStack.length - 1] : null), }, - require: { - builtin: ["path"], - context: "sandbox", - external: true, - root: compileConfig.projectDir, - resolve: (moduleName, parentDirName) => - path.join( - parentDirName, - path.relative(parentDirName, compileConfig.projectDir), - moduleName, - ), - }, - sourceExtensions: ["js", "sql", "sqlx", "yaml", "yml"], + builtinModules: ["path"], + resolve: (moduleName, parentDirName) => + path.join(parentDirName, path.relative(parentDirName, compileConfig.projectDir), moduleName), + sourceExtensions: ["js", "sql", "sqlx", "yaml", "yml", "ipynb", "md"], compiler: (code, filePath) => { let source = code; if (needsCallerFileShim && filePath === coreBundlePath) { diff --git a/cli/vm/jit_worker.ts b/cli/vm/jit_worker.ts index 58c9f37e5..1b48b7ec3 100644 --- a/cli/vm/jit_worker.ts +++ b/cli/vm/jit_worker.ts @@ -1,7 +1,7 @@ import * as fs from "fs"; import * as path from "path"; -import { NodeVM } from "vm2"; +import { VmRunner } from "df/common/vm/vm_runner"; import { dataform } from "df/protos/ts"; const pendingRpcCallbacks = new Map< @@ -85,18 +85,14 @@ export async function handleJitRequest(message: { request: any; projectDir: stri const vmFileName = path.resolve(projectDir, "index.js"); - const vm = new NodeVM({ - require: { - builtin: [], - context: "sandbox", - external: { modules: ["@dataform/*"], transitive: false }, - root: projectDir, - mock: hasProjectLocalCore - ? {} - : { - "@dataform/core": require("@dataform/core"), - }, - }, + const vm = new VmRunner({ + projectDir, + builtinModules: [], + mockModules: hasProjectLocalCore + ? {} + : { + "@dataform/core": require("@dataform/core"), + }, sourceExtensions: ["js", "json", "yaml", "yml"], }); diff --git a/common/vm/BUILD b/common/vm/BUILD new file mode 100644 index 000000000..7203034bf --- /dev/null +++ b/common/vm/BUILD @@ -0,0 +1,50 @@ +package(default_visibility = ["//visibility:public"]) + +load("//testing:index.bzl", "ts_test_suite") +load("//tools:ts_library.bzl", "ts_library") + +ts_library( + name = "vm_runner", + srcs = [ + "vm_runner.ts", + ], + deps = [ + "@npm//@types/node", + ], +) + +ts_test_suite( + name = "tests", + srcs = ["vm_runner_test.ts"], + deps = [ + ":vm_runner", + "//testing", + "@npm//@types/chai", + "@npm//@types/node", + "@npm//chai", + ], +) + +load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary") + +ts_library( + name = "benchmark_lib", + srcs = ["vm_runner_benchmark.ts"], + deps = [ + ":vm_runner", + "@npm//@types/node", + ], +) + +nodejs_binary( + name = "benchmark", + data = [ + ":benchmark_lib", + "@npm//source-map-support", + ], + entry_point = ":vm_runner_benchmark.ts", + templated_args = [ + "--node_options=--require=source-map-support/register", + "--bazel_patch_module_resolver", + ], +) diff --git a/common/vm/vm_runner.ts b/common/vm/vm_runner.ts new file mode 100644 index 000000000..612e6c3d9 --- /dev/null +++ b/common/vm/vm_runner.ts @@ -0,0 +1,412 @@ +import * as fs from "fs"; +import { builtinModules as nodeBuiltins, createRequire } from "module"; +import * as path from "path"; +import * as vm from "vm"; + +export type CompilerFunction = (code: string, filePath: string) => string; + +export interface VmRunnerOptions { + projectDir: string; + sourceExtensions?: string[]; + compiler?: CompilerFunction; + sandbox?: Record; + builtinModules?: string[]; + mockModules?: Record; + resolve?: (moduleName: string, parentDirName: string) => string; + console?: "inherit" | "off"; + env?: Record; + envAllowlist?: string[]; + allowedExternalPaths?: string[]; +} + +export class VmRunner { + private readonly projectDir: string; + private readonly allowedExternalPaths: string[]; + private readonly sourceExtensions: Set; + private readonly allExtensions: string[]; + private readonly compiler?: CompilerFunction; + private readonly builtinModules: Set; + private readonly mockModules: Record; + private readonly customResolve?: (moduleName: string, parentDirName: string) => string; + private readonly context: vm.Context; + private readonly moduleCache = new Map< + string, + { exports: any; id: string; filename: string; loaded: boolean } + >(); + private readonly resolveCache = new Map(); + private readonly nodeBuiltinSet: Set; + + constructor(options: VmRunnerOptions) { + this.projectDir = this.getRealPath(options.projectDir); + this.allowedExternalPaths = (options.allowedExternalPaths || []).map(p => this.getRealPath(p)); + const rawExtensions = options.sourceExtensions || ["js", "json"]; + this.sourceExtensions = new Set( + rawExtensions.map(ext => (ext.startsWith(".") ? ext.slice(1).toLowerCase() : ext.toLowerCase())) + ); + this.allExtensions = Array.from( + new Set([ + ".js", + ".json", + ...rawExtensions.map(ext => (ext.startsWith(".") ? ext : `.${ext}`)) + ]) + ); + this.compiler = options.compiler; + this.builtinModules = new Set(options.builtinModules !== undefined ? options.builtinModules : ["path"]); + this.mockModules = options.mockModules || {}; + this.customResolve = options.resolve; + this.nodeBuiltinSet = new Set(nodeBuiltins); + + let env: Record; + if (options.env !== undefined) { + env = { ...options.env }; + } else if (options.envAllowlist !== undefined) { + env = {}; + for (const key of options.envAllowlist) { + if (key in process.env) { + env[key] = process.env[key]; + } + } + } else { + env = { ...process.env }; + } + + const sandbox: Record = { + console: + options.console === "off" + ? { log: () => {}, error: () => {}, warn: () => {}, info: () => {} } + : console, + process: { + env, + cwd: () => this.projectDir, + version: process.version, + versions: process.versions, + platform: process.platform, + arch: process.arch + }, + Buffer, + Uint8Array, + ArrayBuffer, + setTimeout, + clearTimeout, + setInterval, + clearInterval, + setImmediate, + clearImmediate, + URL, + URLSearchParams, + TextEncoder, + TextDecoder, + ...(options.sandbox || {}) + }; + + this.context = vm.createContext(sandbox); + sandbox.global = this.context; + sandbox.globalThis = this.context; + } + + public run(code: string, filename: string = path.join(this.projectDir, "index.js")): any { + let source = code; + const ext = path.extname(filename).toLowerCase().replace(/^\./, ""); + if (ext === "json") { + const module = { + exports: JSON.parse(source), + id: filename, + filename, + loaded: true + }; + this.moduleCache.set(filename, module); + return module.exports; + } + + if (this.compiler && this.sourceExtensions.has(ext)) { + source = this.compiler(source, filename); + } + + const fn = vm.compileFunction( + source, + ["exports", "require", "module", "__filename", "__dirname"], + { + filename, + parsingContext: this.context + } + ); + + const module = { + exports: {}, + id: filename, + filename, + loaded: false + }; + this.moduleCache.set(filename, module); + + try { + const scopedRequire = this.createRequire(filename); + const result = fn.call( + module.exports, + module.exports, + scopedRequire, + module, + filename, + path.dirname(filename) + ); + module.loaded = true; + + return result !== undefined ? result : module.exports; + } catch (e) { + this.moduleCache.delete(filename); + throw e; + } + } + + public require( + moduleName: string, + fromPath: string = path.join(this.projectDir, "index.js") + ): any { + if (Object.prototype.hasOwnProperty.call(this.mockModules, moduleName)) { + return this.mockModules[moduleName]; + } + + const cleanBuiltinName = moduleName.startsWith("node:") ? moduleName.slice(5) : moduleName; + if (this.nodeBuiltinSet.has(cleanBuiltinName)) { + if (this.builtinModules.has(cleanBuiltinName) || this.builtinModules.has(moduleName)) { + return require(moduleName); + } + const err: any = new Error( + `Access to built-in module '${moduleName}' is not allowed` + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + + const resolvedPath = this.resolve(moduleName, fromPath); + if (this.moduleCache.has(resolvedPath)) { + return this.moduleCache.get(resolvedPath)!.exports; + } + + const module = { + exports: {}, + id: resolvedPath, + filename: resolvedPath, + loaded: false + }; + this.moduleCache.set(resolvedPath, module); + + try { + const ext = path.extname(resolvedPath).toLowerCase().replace(/^\./, ""); + if (ext === "json") { + const content = fs.readFileSync(resolvedPath, "utf8"); + module.exports = JSON.parse(content); + module.loaded = true; + return module.exports; + } + + let source = fs.readFileSync(resolvedPath, "utf8"); + if (this.compiler && this.sourceExtensions.has(ext)) { + source = this.compiler(source, resolvedPath); + } + + const fn = vm.compileFunction( + source, + ["exports", "require", "module", "__filename", "__dirname"], + { + filename: resolvedPath, + parsingContext: this.context + } + ); + + const scopedRequire = this.createRequire(resolvedPath); + fn.call( + module.exports, + module.exports, + scopedRequire, + module, + resolvedPath, + path.dirname(resolvedPath) + ); + module.loaded = true; + + return module.exports; + } catch (e) { + this.moduleCache.delete(resolvedPath); + throw e; + } + } + + public resolve(moduleName: string, fromPath: string): string { + const cacheKey = `${fromPath}\0${moduleName}`; + if (this.resolveCache.has(cacheKey)) { + return this.resolveCache.get(cacheKey)!; + } + + const parentDir = path.dirname(fromPath); + + // Check custom resolve function if provided + if (this.customResolve) { + try { + const candidate = this.customResolve(moduleName, parentDir); + const resolved = this.tryResolvePath(candidate); + if (resolved) { + this.resolveCache.set(cacheKey, resolved); + return resolved; + } + } catch {} + } + + // Relative or absolute path + if (moduleName.startsWith("./") || moduleName.startsWith("../") || path.isAbsolute(moduleName)) { + const candidate = path.resolve(parentDir, moduleName); + const resolved = this.tryResolvePath(candidate); + if (resolved) { + if (!this.isPathContained(resolved)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'` + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + this.resolveCache.set(cacheKey, resolved); + return resolved; + } + } else { + // Project-relative path (e.g. require("includes/helpers")) + const projectRelative = path.resolve(this.projectDir, moduleName); + const resolvedProjectRelative = this.tryResolvePath(projectRelative); + if (resolvedProjectRelative) { + if (!this.isPathContained(resolvedProjectRelative)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'` + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + this.resolveCache.set(cacheKey, resolvedProjectRelative); + return resolvedProjectRelative; + } + + // Check project node_modules directory directly (e.g. @dataform/core) + const nodeModulesCandidate = path.resolve(this.projectDir, "node_modules", moduleName); + const resolvedNodeModules = this.tryResolvePath(nodeModulesCandidate); + if (resolvedNodeModules) { + this.resolveCache.set(cacheKey, resolvedNodeModules); + return resolvedNodeModules; + } + + // Fallback to standard Node.js require.resolve resolution + let nodeReqError: any; + try { + const nodeReq = createRequire(fromPath); + const resolved = nodeReq.resolve(moduleName); + this.resolveCache.set(cacheKey, resolved); + return resolved; + } catch (e) { + nodeReqError = e; + } + + let projectReqError: any; + try { + const projectReq = createRequire(path.join(this.projectDir, "index.js")); + const resolved = projectReq.resolve(moduleName); + this.resolveCache.set(cacheKey, resolved); + return resolved; + } catch (e) { + projectReqError = e; + } + + const err: any = new Error(`Cannot find module '${moduleName}' from '${fromPath}'`); + err.code = "MODULE_NOT_FOUND"; + if (nodeReqError || projectReqError) { + err.cause = nodeReqError || projectReqError; + } + throw err; + } + + const err: any = new Error(`Cannot find module '${moduleName}' from '${fromPath}'`); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + + private getRealPath(targetPath: string): string { + try { + return fs.realpathSync(targetPath); + } catch { + return path.resolve(targetPath); + } + } + + private isPathContained(targetPath: string): boolean { + const realTarget = this.getRealPath(targetPath); + const isContainedIn = (parentDir: string) => { + const rel = path.relative(parentDir, realTarget); + return !rel.startsWith("..") && !path.isAbsolute(rel); + }; + + if (isContainedIn(this.projectDir)) { + return true; + } + return this.allowedExternalPaths.some(allowed => isContainedIn(allowed)); + } + + private getStat(targetPath: string): fs.Stats | null { + try { + return fs.statSync(targetPath); + } catch { + return null; + } + } + + private tryResolvePath(candidatePath: string): string | null { + const stat = this.getStat(candidatePath); + if (stat) { + if (stat.isFile()) { + return candidatePath; + } + if (stat.isDirectory()) { + const pkgPath = path.join(candidatePath, "package.json"); + const pkgStat = this.getStat(pkgPath); + if (pkgStat && pkgStat.isFile()) { + try { + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); + if (pkg.main) { + const mainPath = path.resolve(candidatePath, pkg.main); + const resolvedMain = this.tryResolvePath(mainPath); + if (resolvedMain) { + return resolvedMain; + } + } + } catch {} + } + for (const ext of this.allExtensions) { + const indexPath = path.join(candidatePath, `index${ext}`); + const indexStat = this.getStat(indexPath); + if (indexStat && indexStat.isFile()) { + return indexPath; + } + } + } + } + + for (const ext of this.allExtensions) { + const withExt = candidatePath.endsWith(ext) ? candidatePath : `${candidatePath}${ext}`; + const withExtStat = this.getStat(withExt); + if (withExtStat && withExtStat.isFile()) { + return withExt; + } + } + + return null; + } + + private createRequire(fromPath: string): NodeJS.Require { + const requireFn = ((moduleName: string) => { + return this.require(moduleName, fromPath); + }) as any; + + requireFn.resolve = (moduleName: string) => { + return this.resolve(moduleName, fromPath); + }; + requireFn.extensions = {}; + requireFn.main = undefined; + + return requireFn; + } +} diff --git a/common/vm/vm_runner_benchmark.ts b/common/vm/vm_runner_benchmark.ts new file mode 100644 index 000000000..f2f444154 --- /dev/null +++ b/common/vm/vm_runner_benchmark.ts @@ -0,0 +1,46 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { VmRunner } from "df/common/vm/vm_runner"; + +function runBenchmark() { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "vm-runner-benchmark-")); + try { + fs.mkdirSync(path.join(tmpDir, "includes")); + fs.writeFileSync( + path.join(tmpDir, "includes", "helpers.js"), + "module.exports = { format: (x) => 'formatted_' + x };" + ); + + for (let i = 0; i < 50; i++) { + fs.writeFileSync( + path.join(tmpDir, `table_${i}.js`), + `const { format } = require("./includes/helpers"); + module.exports = { name: format("table_${i}"), query: "SELECT ${i}" };` + ); + } + + const runner = new VmRunner({ projectDir: tmpDir }); + + const iterations = 500; + const start = process.hrtime.bigint(); + + for (let iter = 0; iter < iterations; iter++) { + const idx = iter % 50; + runner.run(`require("./table_${idx}");`); + } + + const end = process.hrtime.bigint(); + const durationMs = Number(end - start) / 1e6; + const opsPerSec = Math.round(iterations / (durationMs / 1000)); + + // eslint-disable-next-line no-console + console.log("VmRunner Benchmark Results:"); + // eslint-disable-next-line no-console + console.log(` Evaluated ${iterations} module requires in ${durationMs.toFixed(2)} ms (${opsPerSec} ops/sec)`); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +} + +runBenchmark(); diff --git a/common/vm/vm_runner_test.ts b/common/vm/vm_runner_test.ts new file mode 100644 index 000000000..71df213bf --- /dev/null +++ b/common/vm/vm_runner_test.ts @@ -0,0 +1,372 @@ +import { expect } from "chai"; +import * as fs from "fs"; +import * as path from "path"; +import { VmRunner } from "df/common/vm/vm_runner"; +import { suite, test } from "df/testing"; +import { TmpDirFixture } from "df/testing/fixtures"; + +suite("VmRunner", ({ afterEach }) => { + const tmpDirFixture = new TmpDirFixture(afterEach); + + test("executes basic script and returns return value", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run("return 40 + 2;"); + expect(result).to.equal(42); + }); + + test("returns module.exports when no explicit return statement exists", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run("module.exports = { value: 'hello' };"); + expect(result).to.deep.equal({ value: "hello" }); + }); + + test("parses JSON files in run method when filename has .json extension", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run( + JSON.stringify({ name: "dataform-test", active: true }), + path.join(tmpDir, "config.json") + ); + expect(result).to.deep.equal({ name: "dataform-test", active: true }); + }); + + test("resolves relative requires and json files", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(tmpDir, "config.json"), JSON.stringify({ name: "test-project" })); + fs.mkdirSync(path.join(tmpDir, "sub")); + fs.writeFileSync( + path.join(tmpDir, "sub", "helper.js"), + "module.exports = { greet: (x) => `Hello ${x}` };" + ); + + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run(` + const config = require("./config.json"); + const { greet } = require("./sub/helper"); + return greet(config.name); + `); + expect(result).to.equal("Hello test-project"); + }); + + test("resolves project-relative requires (without leading ./)", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.mkdirSync(path.join(tmpDir, "includes")); + fs.writeFileSync( + path.join(tmpDir, "includes", "math.js"), + "module.exports = { add: (a, b) => a + b };" + ); + + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run(` + const math = require("includes/math"); + return math.add(10, 20); + `); + expect(result).to.equal(30); + }); + + test("applies compiler hook to custom sourceExtensions", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(tmpDir, "model.sqlx"), + "SELECT 1 AS id" + ); + + const runner = new VmRunner({ + projectDir: tmpDir, + sourceExtensions: ["js", "sqlx"], + compiler: (code, filePath) => { + if (filePath.endsWith(".sqlx")) { + return `module.exports = { query: ${JSON.stringify(code.trim())}, file: ${JSON.stringify(filePath)} };`; + } + return code; + } + }); + + const result = runner.run(` + const model = require("./model.sqlx"); + return model; + `); + expect(result.query).to.equal("SELECT 1 AS id"); + expect(result.file).to.equal(path.join(tmpDir, "model.sqlx")); + }); + + test("handles circular require without crashing", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(tmpDir, "a.js"), + ` + exports.name = "moduleA"; + const b = require("./b"); + exports.getBName = () => b.name; + ` + ); + fs.writeFileSync( + path.join(tmpDir, "b.js"), + ` + exports.name = "moduleB"; + const a = require("./a"); + exports.getAName = () => a.name; + ` + ); + + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run(` + const a = require("./a"); + const b = require("./b"); + return { aToB: a.getBName(), bToA: b.getAName() }; + `); + expect(result.aToB).to.equal("moduleB"); + expect(result.bToA).to.equal("moduleA"); + }); + + test("allows configured builtin modules and rejects unallowed ones", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ + projectDir: tmpDir, + builtinModules: ["path"] + }); + + const pathResult = runner.run(` + const path = require("path"); + return path.join("foo", "bar"); + `); + expect(pathResult).to.equal(path.join("foo", "bar")); + + expect(() => { + runner.run(`require("fs");`); + }).to.throw(/Access to built-in module 'fs' is not allowed/); + }); + + test("intercepts mockModules", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const mockCore = { + version: "9.9.9", + compiler: () => "compiled" + }; + + const runner = new VmRunner({ + projectDir: tmpDir, + mockModules: { + "@dataform/core": mockCore + } + }); + + const result = runner.run(` + const core = require("@dataform/core"); + return core.version; + `); + expect(result).to.equal("9.9.9"); + }); + + test("does not pollute host global", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ projectDir: tmpDir }); + + runner.run(`global.pollutedState = "in-sandbox";`); + expect((global as any).pollutedState).to.equal(undefined); + }); + + test("retains context global state across run calls", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ + projectDir: tmpDir, + sandbox: { + injectedValue: 123 + } + }); + + const result = runner.run(` + global.customState = "active"; + return injectedValue + 1; + `); + expect(result).to.equal(124); + + const state = runner.run("return global.customState;"); + expect(state).to.equal("active"); + }); + + test("removes module from cache when module execution throws and re-throws on next require", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const brokenFile = path.join(tmpDir, "broken.js"); + fs.writeFileSync(brokenFile, "throw new Error('boom');"); + + const runner = new VmRunner({ projectDir: tmpDir }); + + expect(() => runner.run(`require("./broken");`)).to.throw("boom"); + // Ensure second require also throws and does not return an empty cached exports object + expect(() => runner.run(`require("./broken");`)).to.throw("boom"); + }); + + test("rejects requires that escape projectDir via relative or absolute path traversal", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const outsideDir = tmpDirFixture.createNewTmpDir(); + const secretFile = path.join(outsideDir, "secret.json"); + fs.writeFileSync(secretFile, JSON.stringify({ secret: "sensitive" })); + + const runner = new VmRunner({ projectDir: tmpDir }); + + // Relative path traversal + const relativePath = path.relative(tmpDir, secretFile); + expect(() => runner.run(`require(${JSON.stringify(relativePath)});`)).to.throw( + /outside of project directory/ + ); + + // Absolute path traversal + expect(() => runner.run(`require(${JSON.stringify(secretFile)});`)).to.throw( + /outside of project directory/ + ); + }); + + test("allows requiring external files when explicitly permitted via allowedExternalPaths", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const sharedDir = tmpDirFixture.createNewTmpDir(); + const sharedFile = path.join(sharedDir, "shared.json"); + fs.writeFileSync(sharedFile, JSON.stringify({ shared: "data" })); + + const runner = new VmRunner({ + projectDir: tmpDir, + allowedExternalPaths: [sharedDir] + }); + + const result = runner.run(` + const data = require(${JSON.stringify(sharedFile)}); + return data.shared; + `); + expect(result).to.equal("data"); + }); + + test("supports custom env and envAllowlist", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + process.env.TEST_HOST_SECRET = "secret_123"; + process.env.TEST_PUBLIC_VAR = "public_abc"; + + try { + // With envAllowlist + const allowlistRunner = new VmRunner({ + projectDir: tmpDir, + envAllowlist: ["TEST_PUBLIC_VAR"] + }); + const allowlistEnv = allowlistRunner.run("return process.env;"); + expect(allowlistEnv.TEST_PUBLIC_VAR).to.equal("public_abc"); + expect(allowlistEnv.TEST_HOST_SECRET).to.equal(undefined); + + // With custom env record + const customRunner = new VmRunner({ + projectDir: tmpDir, + env: { CUSTOM_KEY: "custom_value" } + }); + const customEnv = customRunner.run("return process.env;"); + expect(customEnv.CUSTOM_KEY).to.equal("custom_value"); + expect(customEnv.TEST_PUBLIC_VAR).to.equal(undefined); + } finally { + delete process.env.TEST_HOST_SECRET; + delete process.env.TEST_PUBLIC_VAR; + } + }); + + test("compiles and requires .ipynb and .md files via compiler hook", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(tmpDir, "notebook.ipynb"), + JSON.stringify({ cells: [{ cell_type: "code", source: ["print('hello')"] }] }) + ); + fs.writeFileSync( + path.join(tmpDir, "doc.md"), + "# Hello Documentation" + ); + + const runner = new VmRunner({ + projectDir: tmpDir, + sourceExtensions: ["js", "json", "ipynb", "md"], + compiler: (code, filePath) => { + if (filePath.endsWith(".ipynb")) { + return `module.exports = { asJson: JSON.parse(${JSON.stringify(code)}) };`; + } + if (filePath.endsWith(".md")) { + return `module.exports = { asMarkdown: ${JSON.stringify(code)} };`; + } + return code; + } + }); + + const result = runner.run(` + const notebook = require("./notebook.ipynb"); + const doc = require("./doc.md"); + return { + cellType: notebook.asJson.cells[0].cell_type, + docTitle: doc.asMarkdown.trim() + }; + `); + expect(result.cellType).to.equal("code"); + expect(result.docTitle).to.equal("# Hello Documentation"); + }); + + test("preserves V8 CallSite file paths and line numbers in stack traces", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const errorFile = path.join(tmpDir, "faulty.sqlx"); + fs.writeFileSync(errorFile, "throw new Error('boom');"); + + const runner = new VmRunner({ + projectDir: tmpDir, + sourceExtensions: ["sqlx"], + compiler: code => code + }); + + let caughtError: Error | null = null; + try { + runner.run(`require("./faulty.sqlx");`); + } catch (e) { + caughtError = e; + } + + expect(caughtError).to.not.equal(null); + expect(caughtError!.stack).to.include(errorFile); + }); + + test("caches module resolution results across multiple requires", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const helperFile = path.join(tmpDir, "helper.js"); + fs.writeFileSync(helperFile, "module.exports = { count: 1 };"); + + const runner = new VmRunner({ projectDir: tmpDir }); + const resolvedFirst = runner.resolve("./helper", path.join(tmpDir, "index.js")); + const resolvedSecond = runner.resolve("./helper", path.join(tmpDir, "index.js")); + expect(resolvedFirst).to.equal(helperFile); + expect(resolvedSecond).to.equal(helperFile); + }); + + test("shares Uint8Array constructor with host across realm boundary", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + let receivedBytes: any = null; + const runner = new VmRunner({ + projectDir: tmpDir, + mockModules: { + "@dataform/core": { + jitCompiler: () => ({ + compile: (bytes: Uint8Array) => { + receivedBytes = bytes; + return new Uint8Array([bytes[0] + 1, bytes[1] + 1]); + } + }) + } + } + }); + + const result = runner.run(` + const { jitCompiler } = require("@dataform/core"); + const compiler = jitCompiler(); + const input = new Uint8Array([10, 20]); + const output = compiler.compile(input); + module.exports = { input, output }; + `); + + expect(receivedBytes).to.be.an.instanceOf(Uint8Array); + expect(receivedBytes[0]).to.equal(10); + expect(result.input).to.be.an.instanceOf(Uint8Array); + expect(result.output).to.be.an.instanceOf(Uint8Array); + expect(Array.from(result.output)).to.deep.equal([11, 21]); + }); +}); + diff --git a/core/main_property_graphs_test.ts b/core/main_property_graphs_test.ts index 1bd7fb4c3..468b9ccb6 100644 --- a/core/main_property_graphs_test.ts +++ b/core/main_property_graphs_test.ts @@ -36,13 +36,19 @@ suite("property graphs", ({ afterEach }) => { defaultDatabase: "defaultProject", defaultLocation: "US", }; - const graphStackTail = "\n at CallSite {}".repeat(10); const graphError = (fileName: string, message: string, extra: object = {}) => ({ fileName, message, - stack: `Error: ${message}${graphStackTail}`, ...extra, }); + const asPlainGraph = (graph: dataform.ICompiledGraph) => { + const plain = asPlainObject(graph); + plain.graphErrors?.compilationErrors?.forEach((e: any) => { + expect(e.stack).to.include(`Error: ${e.message}`); + delete e.stack; + }); + return plain; + }; const missingRefTarget = { schema: "defaultDataset", @@ -1605,7 +1611,7 @@ entities: } if (testParameters.expectedGraph) { - expect(asPlainObject(result.compile?.compiledGraph)).deep.equals( + expect(asPlainGraph(result.compile?.compiledGraph)).deep.equals( asPlainObject(testParameters.expectedGraph), ); } @@ -1647,7 +1653,7 @@ entities: const result = runMainInVm(request); - expect(asPlainObject(result.compile?.compiledGraph)).deep.equals( + expect(asPlainGraph(result.compile?.compiledGraph)).deep.equals( asPlainObject({ projectConfig: graphProjectConfig, graphErrors: { diff --git a/package.json b/package.json index 1000b614b..a296193c7 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,6 @@ "typeid-js": "0.3.0", "untildify": "^4.0.0", "url": "^0.11.0", - "vm2": "3.11.6", "vscode-jsonrpc": "^5.0.1", "vscode-languageclient": "^6.1.3", "vscode-languageserver": "^6.1.1", diff --git a/packages/@dataform/cli/BUILD b/packages/@dataform/cli/BUILD index 31309c581..cf4eea43b 100644 --- a/packages/@dataform/cli/BUILD +++ b/packages/@dataform/cli/BUILD @@ -51,7 +51,6 @@ externals = [ "tmp", "typeid-js", "untildify", - "vm2", "yargs", ] diff --git a/packages/rollup.config.js b/packages/rollup.config.js index 3ac33cdec..a88828070 100644 --- a/packages/rollup.config.js +++ b/packages/rollup.config.js @@ -20,7 +20,9 @@ const knownNodeBuiltins = [ "events", "long", "https", - "net" + "net", + "module", + "vm" ].map(moduleName => convertToRegex(moduleName)); const importsToBundle = ["df", /df\/.*$/, /^bazel\-.*$/]; diff --git a/testing/BUILD b/testing/BUILD index d0ce2ae5c..42025fdfd 100644 --- a/testing/BUILD +++ b/testing/BUILD @@ -39,12 +39,12 @@ ts_library( ], deps = [ "//common/protos", + "//common/vm:vm_runner", "//core", "//protos:ts", "@npm//@types/fs-extra", "@npm//@types/node", "@npm//fs-extra", - "@npm//vm2", ], ) diff --git a/testing/run_core.ts b/testing/run_core.ts index f627bd49c..c970b2bcb 100644 --- a/testing/run_core.ts +++ b/testing/run_core.ts @@ -1,8 +1,8 @@ import * as fs from "fs-extra"; import * as path from "path"; -import { CompilerFunction, NodeVM } from "vm2"; import { decode64, encode64 } from "df/common/protos"; +import { VmRunner } from "df/common/vm/vm_runner"; import { compile } from "df/core/compilers"; import { dataform } from "df/protos/ts"; @@ -74,17 +74,16 @@ export function runMainInVm( // Copy over the build Dataform Core that is set up as a node_modules directory. fs.copySync(`${process.cwd()}/core/node_modules`, `${projectDir}/node_modules`); - const compiler = compile as CompilerFunction; + const compiler = compile; // See cli/vm/compile.ts for why we use a host-side stack + enter/exit helpers // instead of writing to `global.__dataform_current_file` inside every module. const fileStack: string[] = []; - // Then use vm2's native compiler integration to apply the compiler to files. - const nodeVm = new NodeVM({ + const vmRunner = new VmRunner({ + projectDir, // Inheriting the console makes console.logs show when tests are running, which is useful for // debugging. console: "inherit", - wrapper: "none", sandbox: { __df_enter: (p: string) => { fileStack.push(p); @@ -94,14 +93,9 @@ export function runMainInVm( }, __df_current: () => (fileStack.length > 0 ? fileStack[fileStack.length - 1] : null), }, - require: { - builtin: ["path"], - context: "sandbox", - external: true, - root: projectDir, - resolve: (moduleName, parentDirName) => - path.join(parentDirName, path.relative(parentDirName, projectDir), moduleName), - }, + builtinModules: ["path"], + resolve: (moduleName, parentDirName) => + path.join(parentDirName, path.relative(parentDirName, projectDir), moduleName), sourceExtensions: SOURCE_EXTENSIONS, compiler: (code, filePath) => { const compiledCode = compiler(code, filePath); @@ -116,16 +110,23 @@ export function runMainInVm( }, }); + const hasWorkflowSettingsYaml = fs.existsSync( + path.join(projectDir, "workflow_settings.yaml") + ); + const hasDataformJson = fs.existsSync( + path.join(projectDir, "dataform.json") + ); + const encodedCoreExecutionRequest = encode64(dataform.CoreExecutionRequest, coreExecutionRequest); const vmIndexFileName = path.resolve(path.join(projectDir, "index.js")); - const encodedCoreExecutionResponse = nodeVm.run( + const encodedCoreExecutionResponse = vmRunner.run( ` Object.defineProperty(global, '__dataform_current_file', { configurable: true, get: function() { return __df_current(); } }); - global.workflowSettingsYaml = (function() { try { return require("./workflow_settings.yaml"); } catch(e) { console.error("YAML require failed run_core:", e); } })(); - global.dataformJson = (function() { try { return require("./dataform.json"); } catch(e) {} })(); + ${hasWorkflowSettingsYaml ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' : ''} + ${hasDataformJson ? 'global.dataformJson = require("./dataform.json");' : ''} return require("@dataform/core").main("${encodedCoreExecutionRequest}") `, vmIndexFileName, diff --git a/yarn.lock b/yarn.lock index 9b04612d6..10bb33ecd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -863,13 +863,6 @@ acorn-jsx@^5.3.1, acorn-jsx@^5.3.2: resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity "sha1-ftW7VZCLOy8bxVxq8WU7rafweTc= sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==" -acorn-walk@^8.3.4: - version "8.3.4" - resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.3.4.tgz#794dd169c3977edf4ba4ea47583587c5866236b7" - integrity sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g== - dependencies: - acorn "^8.11.0" - acorn@^7.4.0: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" @@ -3459,14 +3452,6 @@ uuidv7@^0.4.4: resolved "https://registry.yarnpkg.com/uuidv7/-/uuidv7-0.4.4.tgz#e7ffd7981f590c478fb8868eff4bb3bc55fa90e6" integrity sha512-jjRGChg03uGp9f6wQYSO8qXkweJwRbA5WRuEQE8xLIiehIzIIi23qZSzsyvZPCPoFqkeLtZuz7Plt1LGukAInA== -vm2@3.11.6: - version "3.11.6" - resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.11.6.tgz#044ddbbd68c0157bc07b2e2fca20f38d3d673be7" - integrity sha512-35hVTcKieg7jJMntHhgWT5c2a1J2vmpXm66Xs1Z8ayHOJTj8rzIlXKzba3nO1Om/sonmnkjlAOMt9p8lYJXsWw== - dependencies: - acorn "^8.15.0" - acorn-walk "^8.3.4" - vscode-jsonrpc@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/vscode-jsonrpc/-/vscode-jsonrpc-5.0.1.tgz#9bab9c330d89f43fc8c1e8702b5c36e058a01794" From d81abda3bc7b049539462aa67d7ce62543949c31 Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Thu, 17 Sep 2026 17:41:20 +0000 Subject: [PATCH 2/4] Enforce path containment in resolver and drop redundant resolve hooks --- cli/vm/compile.ts | 2 -- common/vm/vm_runner.ts | 43 ++++++++++++++++++++++++++++++++++++- common/vm/vm_runner_test.ts | 33 ++++++++++++++++++++++++++++ testing/run_core.ts | 2 -- 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index 7a0c1d911..ba74c3596 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -85,8 +85,6 @@ export function compile(compileConfig: dataform.ICompileConfig) { __df_current: () => (fileStack.length > 0 ? fileStack[fileStack.length - 1] : null), }, builtinModules: ["path"], - resolve: (moduleName, parentDirName) => - path.join(parentDirName, path.relative(parentDirName, compileConfig.projectDir), moduleName), sourceExtensions: ["js", "sql", "sqlx", "yaml", "yml", "ipynb", "md"], compiler: (code, filePath) => { let source = code; diff --git a/common/vm/vm_runner.ts b/common/vm/vm_runner.ts index 612e6c3d9..41aec0a54 100644 --- a/common/vm/vm_runner.ts +++ b/common/vm/vm_runner.ts @@ -246,10 +246,21 @@ export class VmRunner { const candidate = this.customResolve(moduleName, parentDir); const resolved = this.tryResolvePath(candidate); if (resolved) { + if (!this.isPathContained(resolved)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } this.resolveCache.set(cacheKey, resolved); return resolved; } - } catch {} + } catch (e) { + if (e && e.code === "MODULE_NOT_FOUND") { + throw e; + } + } } // Relative or absolute path @@ -296,9 +307,24 @@ export class VmRunner { try { const nodeReq = createRequire(fromPath); const resolved = nodeReq.resolve(moduleName); + if (!this.isPathContained(resolved)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } this.resolveCache.set(cacheKey, resolved); return resolved; } catch (e) { + if ( + e && + e.code === "MODULE_NOT_FOUND" && + e.message && + e.message.includes("outside of project directory") + ) { + throw e; + } nodeReqError = e; } @@ -306,9 +332,24 @@ export class VmRunner { try { const projectReq = createRequire(path.join(this.projectDir, "index.js")); const resolved = projectReq.resolve(moduleName); + if (!this.isPathContained(resolved)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } this.resolveCache.set(cacheKey, resolved); return resolved; } catch (e) { + if ( + e && + e.code === "MODULE_NOT_FOUND" && + e.message && + e.message.includes("outside of project directory") + ) { + throw e; + } projectReqError = e; } diff --git a/common/vm/vm_runner_test.ts b/common/vm/vm_runner_test.ts index 71df213bf..ceb2defff 100644 --- a/common/vm/vm_runner_test.ts +++ b/common/vm/vm_runner_test.ts @@ -219,6 +219,39 @@ suite("VmRunner", ({ afterEach }) => { ); }); + test("rejects requires that escape projectDir via customResolve", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const outsideDir = tmpDirFixture.createNewTmpDir(); + const secretFile = path.join(outsideDir, "secret.json"); + fs.writeFileSync(secretFile, JSON.stringify({ secret: "sensitive" })); + + const runner = new VmRunner({ + projectDir: tmpDir, + resolve: (moduleName) => path.resolve(outsideDir, moduleName), + }); + + expect(() => runner.run(`require("secret.json");`)).to.throw(/outside of project directory/); + }); + + test("resolves relative paths from subfolders relative to caller directory without custom resolve", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const subDir = path.join(tmpDir, "models", "sub"); + fs.mkdirSync(subDir, { recursive: true }); + + const rootHelper = path.join(tmpDir, "helper.js"); + fs.writeFileSync(rootHelper, "module.exports = 'root';"); + + const subHelper = path.join(subDir, "helper.js"); + fs.writeFileSync(subHelper, "module.exports = 'sub';"); + + const subCaller = path.join(subDir, "caller.js"); + fs.writeFileSync(subCaller, "module.exports = require('./helper');"); + + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.require("./models/sub/caller"); + expect(result).to.equal("sub"); + }); + test("allows requiring external files when explicitly permitted via allowedExternalPaths", () => { const tmpDir = tmpDirFixture.createNewTmpDir(); const sharedDir = tmpDirFixture.createNewTmpDir(); diff --git a/testing/run_core.ts b/testing/run_core.ts index c970b2bcb..9cd0e9a4d 100644 --- a/testing/run_core.ts +++ b/testing/run_core.ts @@ -94,8 +94,6 @@ export function runMainInVm( __df_current: () => (fileStack.length > 0 ? fileStack[fileStack.length - 1] : null), }, builtinModules: ["path"], - resolve: (moduleName, parentDirName) => - path.join(parentDirName, path.relative(parentDirName, projectDir), moduleName), sourceExtensions: SOURCE_EXTENSIONS, compiler: (code, filePath) => { const compiledCode = compiler(code, filePath); From 6c6cd2a5862c81941fbd33d0bd1763b11b03dd4e Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Tue, 22 Sep 2026 14:01:59 +0000 Subject: [PATCH 3/4] Address review comments on VmRunner migration --- cli/vm/compile.ts | 31 ++- cli/vm/jit_worker.ts | 1 + common/vm/vm_runner.ts | 354 ++++++++++++++---------------- common/vm/vm_runner_test.ts | 153 ++++++++++--- core/main_property_graphs_test.ts | 5 + readme.md | 2 + testing/run_core.ts | 12 +- 7 files changed, 318 insertions(+), 240 deletions(-) diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index ba74c3596..991719ec7 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -71,19 +71,21 @@ export function compile(compileConfig: dataform.ICompileConfig) { // executing file via a host-side stack exposed through sandbox helpers, and // expose it as a getter on `global.__dataform_current_file`. const fileStack: string[] = []; + const sandbox: Record = {}; + if (needsCallerFileShim) { + sandbox.__df_enter = (p: string) => { + fileStack.push(p); + }; + sandbox.__df_exit = () => { + fileStack.pop(); + }; + sandbox.__df_current = () => (fileStack.length > 0 ? fileStack[fileStack.length - 1] : null); + } // Then use VmRunner to apply the compiler to files. const userCodeVm = new VmRunner({ projectDir: compileConfig.projectDir, - sandbox: { - __df_enter: (p: string) => { - fileStack.push(p); - }, - __df_exit: () => { - fileStack.pop(); - }, - __df_current: () => (fileStack.length > 0 ? fileStack[fileStack.length - 1] : null), - }, + sandbox, builtinModules: ["path"], sourceExtensions: ["js", "sql", "sqlx", "yaml", "yml", "ipynb", "md"], compiler: (code, filePath) => { @@ -92,6 +94,9 @@ export function compile(compileConfig: dataform.ICompileConfig) { source = patchOldCoreCallerFile(source); } const compiledCode = compiler(source, filePath); + if (!needsCallerFileShim) { + return compiledCode; + } return ` __df_enter(${JSON.stringify(filePath)}); try { @@ -110,10 +115,14 @@ export function compile(compileConfig: dataform.ICompileConfig) { return userCodeVm.run( ` - Object.defineProperty(global, '__dataform_current_file', { + ${ + needsCallerFileShim + ? `Object.defineProperty(global, '__dataform_current_file', { configurable: true, get: function() { return __df_current(); } - }); + });` + : "" + } ${ hasWorkflowSettingsYaml ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' diff --git a/cli/vm/jit_worker.ts b/cli/vm/jit_worker.ts index 1b48b7ec3..2b6197e4c 100644 --- a/cli/vm/jit_worker.ts +++ b/cli/vm/jit_worker.ts @@ -88,6 +88,7 @@ export async function handleJitRequest(message: { request: any; projectDir: stri const vm = new VmRunner({ projectDir, builtinModules: [], + allowedModules: ["@dataform/*"], mockModules: hasProjectLocalCore ? {} : { diff --git a/common/vm/vm_runner.ts b/common/vm/vm_runner.ts index 41aec0a54..dbb76f202 100644 --- a/common/vm/vm_runner.ts +++ b/common/vm/vm_runner.ts @@ -1,3 +1,14 @@ +/** + * @fileoverview VmRunner executes JavaScript code within a Node.js `node:vm` context. + * + * IMPORTANT SECURITY NOTE: + * This class is NOT a security boundary or a sandbox against untrusted code. + * As documented by Node.js, `node:vm` contexts can be escaped and do not isolate + * against malicious code execution. VmRunner is intended solely for module isolation, + * custom module resolution, and scoping execution environments (e.g. Dataform CLI + * compilation and testing) where the code being executed is trusted. + */ + import * as fs from "fs"; import { builtinModules as nodeBuiltins, createRequire } from "module"; import * as path from "path"; @@ -17,11 +28,13 @@ export interface VmRunnerOptions { env?: Record; envAllowlist?: string[]; allowedExternalPaths?: string[]; + allowedModules?: string[]; } export class VmRunner { private readonly projectDir: string; private readonly allowedExternalPaths: string[]; + private readonly allowedModules?: string[]; private readonly sourceExtensions: Set; private readonly allExtensions: string[]; private readonly compiler?: CompilerFunction; @@ -38,20 +51,27 @@ export class VmRunner { constructor(options: VmRunnerOptions) { this.projectDir = this.getRealPath(options.projectDir); - this.allowedExternalPaths = (options.allowedExternalPaths || []).map(p => this.getRealPath(p)); + this.allowedExternalPaths = (options.allowedExternalPaths || []).map((p) => + this.getRealPath(p), + ); + this.allowedModules = options.allowedModules; const rawExtensions = options.sourceExtensions || ["js", "json"]; this.sourceExtensions = new Set( - rawExtensions.map(ext => (ext.startsWith(".") ? ext.slice(1).toLowerCase() : ext.toLowerCase())) + rawExtensions.map((ext) => + ext.startsWith(".") ? ext.slice(1).toLowerCase() : ext.toLowerCase(), + ), ); this.allExtensions = Array.from( new Set([ ".js", ".json", - ...rawExtensions.map(ext => (ext.startsWith(".") ? ext : `.${ext}`)) - ]) + ...rawExtensions.map((ext) => (ext.startsWith(".") ? ext : `.${ext}`)), + ]), ); this.compiler = options.compiler; - this.builtinModules = new Set(options.builtinModules !== undefined ? options.builtinModules : ["path"]); + this.builtinModules = new Set( + options.builtinModules !== undefined ? options.builtinModules : ["path"], + ); this.mockModules = options.mockModules || {}; this.customResolve = options.resolve; this.nodeBuiltinSet = new Set(nodeBuiltins); @@ -67,7 +87,8 @@ export class VmRunner { } } } else { - env = { ...process.env }; + // Default to empty environment to avoid leaking host secrets (credentials, API keys). + env = {}; } const sandbox: Record = { @@ -81,7 +102,7 @@ export class VmRunner { version: process.version, versions: process.versions, platform: process.platform, - arch: process.arch + arch: process.arch, }, Buffer, Uint8Array, @@ -96,7 +117,7 @@ export class VmRunner { URLSearchParams, TextEncoder, TextDecoder, - ...(options.sandbox || {}) + ...(options.sandbox || {}), }; this.context = vm.createContext(sandbox); @@ -105,62 +126,12 @@ export class VmRunner { } public run(code: string, filename: string = path.join(this.projectDir, "index.js")): any { - let source = code; - const ext = path.extname(filename).toLowerCase().replace(/^\./, ""); - if (ext === "json") { - const module = { - exports: JSON.parse(source), - id: filename, - filename, - loaded: true - }; - this.moduleCache.set(filename, module); - return module.exports; - } - - if (this.compiler && this.sourceExtensions.has(ext)) { - source = this.compiler(source, filename); - } - - const fn = vm.compileFunction( - source, - ["exports", "require", "module", "__filename", "__dirname"], - { - filename, - parsingContext: this.context - } - ); - - const module = { - exports: {}, - id: filename, - filename, - loaded: false - }; - this.moduleCache.set(filename, module); - - try { - const scopedRequire = this.createRequire(filename); - const result = fn.call( - module.exports, - module.exports, - scopedRequire, - module, - filename, - path.dirname(filename) - ); - module.loaded = true; - - return result !== undefined ? result : module.exports; - } catch (e) { - this.moduleCache.delete(filename); - throw e; - } + return this.executeModule(code, filename, true); } public require( moduleName: string, - fromPath: string = path.join(this.projectDir, "index.js") + fromPath: string = path.join(this.projectDir, "index.js"), ): any { if (Object.prototype.hasOwnProperty.call(this.mockModules, moduleName)) { return this.mockModules[moduleName]; @@ -171,9 +142,7 @@ export class VmRunner { if (this.builtinModules.has(cleanBuiltinName) || this.builtinModules.has(moduleName)) { return require(moduleName); } - const err: any = new Error( - `Access to built-in module '${moduleName}' is not allowed` - ); + const err: any = new Error(`Access to built-in module '${moduleName}' is not allowed`); err.code = "MODULE_NOT_FOUND"; throw err; } @@ -183,53 +152,8 @@ export class VmRunner { return this.moduleCache.get(resolvedPath)!.exports; } - const module = { - exports: {}, - id: resolvedPath, - filename: resolvedPath, - loaded: false - }; - this.moduleCache.set(resolvedPath, module); - - try { - const ext = path.extname(resolvedPath).toLowerCase().replace(/^\./, ""); - if (ext === "json") { - const content = fs.readFileSync(resolvedPath, "utf8"); - module.exports = JSON.parse(content); - module.loaded = true; - return module.exports; - } - - let source = fs.readFileSync(resolvedPath, "utf8"); - if (this.compiler && this.sourceExtensions.has(ext)) { - source = this.compiler(source, resolvedPath); - } - - const fn = vm.compileFunction( - source, - ["exports", "require", "module", "__filename", "__dirname"], - { - filename: resolvedPath, - parsingContext: this.context - } - ); - - const scopedRequire = this.createRequire(resolvedPath); - fn.call( - module.exports, - module.exports, - scopedRequire, - module, - resolvedPath, - path.dirname(resolvedPath) - ); - module.loaded = true; - - return module.exports; - } catch (e) { - this.moduleCache.delete(resolvedPath); - throw e; - } + const source = fs.readFileSync(resolvedPath, "utf8"); + return this.executeModule(source, resolvedPath, false); } public resolve(moduleName: string, fromPath: string): string { @@ -242,116 +166,79 @@ export class VmRunner { // Check custom resolve function if provided if (this.customResolve) { + let candidate: string | undefined; try { - const candidate = this.customResolve(moduleName, parentDir); - const resolved = this.tryResolvePath(candidate); - if (resolved) { - if (!this.isPathContained(resolved)) { - const err: any = new Error( - `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, - ); - err.code = "MODULE_NOT_FOUND"; - throw err; - } - this.resolveCache.set(cacheKey, resolved); - return resolved; - } + candidate = this.customResolve(moduleName, parentDir); } catch (e) { - if (e && e.code === "MODULE_NOT_FOUND") { + if (e && (e as any).code !== "MODULE_NOT_FOUND") { throw e; } } + if (candidate) { + const resolved = this.tryResolvePath(candidate); + if (resolved) { + return this.checkContainmentAndCache(moduleName, resolved, cacheKey); + } + } } // Relative or absolute path - if (moduleName.startsWith("./") || moduleName.startsWith("../") || path.isAbsolute(moduleName)) { + if ( + moduleName.startsWith("./") || + moduleName.startsWith("../") || + path.isAbsolute(moduleName) + ) { const candidate = path.resolve(parentDir, moduleName); const resolved = this.tryResolvePath(candidate); if (resolved) { - if (!this.isPathContained(resolved)) { - const err: any = new Error( - `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'` - ); - err.code = "MODULE_NOT_FOUND"; - throw err; - } - this.resolveCache.set(cacheKey, resolved); - return resolved; + return this.checkContainmentAndCache(moduleName, resolved, cacheKey); } } else { // Project-relative path (e.g. require("includes/helpers")) const projectRelative = path.resolve(this.projectDir, moduleName); const resolvedProjectRelative = this.tryResolvePath(projectRelative); if (resolvedProjectRelative) { - if (!this.isPathContained(resolvedProjectRelative)) { - const err: any = new Error( - `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'` - ); - err.code = "MODULE_NOT_FOUND"; - throw err; - } - this.resolveCache.set(cacheKey, resolvedProjectRelative); - return resolvedProjectRelative; + return this.checkContainmentAndCache(moduleName, resolvedProjectRelative, cacheKey); + } + + // External module check: if allowedModules is specified, package must be allowed + if (!this.isModuleAllowed(moduleName)) { + const err: any = new Error(`Access to module '${moduleName}' is not allowed`); + err.code = "MODULE_NOT_FOUND"; + throw err; } // Check project node_modules directory directly (e.g. @dataform/core) const nodeModulesCandidate = path.resolve(this.projectDir, "node_modules", moduleName); const resolvedNodeModules = this.tryResolvePath(nodeModulesCandidate); if (resolvedNodeModules) { - this.resolveCache.set(cacheKey, resolvedNodeModules); - return resolvedNodeModules; + return this.checkContainmentAndCache(moduleName, resolvedNodeModules, cacheKey); } // Fallback to standard Node.js require.resolve resolution + let nodeReqResolved: string | undefined; let nodeReqError: any; try { const nodeReq = createRequire(fromPath); - const resolved = nodeReq.resolve(moduleName); - if (!this.isPathContained(resolved)) { - const err: any = new Error( - `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, - ); - err.code = "MODULE_NOT_FOUND"; - throw err; - } - this.resolveCache.set(cacheKey, resolved); - return resolved; + nodeReqResolved = nodeReq.resolve(moduleName); } catch (e) { - if ( - e && - e.code === "MODULE_NOT_FOUND" && - e.message && - e.message.includes("outside of project directory") - ) { - throw e; - } nodeReqError = e; } + if (nodeReqResolved) { + return this.checkContainmentAndCache(moduleName, nodeReqResolved, cacheKey); + } + let projectReqResolved: string | undefined; let projectReqError: any; try { const projectReq = createRequire(path.join(this.projectDir, "index.js")); - const resolved = projectReq.resolve(moduleName); - if (!this.isPathContained(resolved)) { - const err: any = new Error( - `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, - ); - err.code = "MODULE_NOT_FOUND"; - throw err; - } - this.resolveCache.set(cacheKey, resolved); - return resolved; + projectReqResolved = projectReq.resolve(moduleName); } catch (e) { - if ( - e && - e.code === "MODULE_NOT_FOUND" && - e.message && - e.message.includes("outside of project directory") - ) { - throw e; - } projectReqError = e; } + if (projectReqResolved) { + return this.checkContainmentAndCache(moduleName, projectReqResolved, cacheKey); + } const err: any = new Error(`Cannot find module '${moduleName}' from '${fromPath}'`); err.code = "MODULE_NOT_FOUND"; @@ -366,6 +253,89 @@ export class VmRunner { throw err; } + private isModuleAllowed(moduleName: string): boolean { + if (!this.allowedModules) { + return true; + } + return this.allowedModules.some((pattern) => { + if (pattern.endsWith("/*")) { + const prefix = pattern.slice(0, -1); + return moduleName.startsWith(prefix); + } + return moduleName === pattern; + }); + } + + private checkContainmentAndCache( + moduleName: string, + resolvedPath: string, + cacheKey: string, + ): string { + if (!this.isPathContained(resolvedPath)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + this.resolveCache.set(cacheKey, resolvedPath); + return resolvedPath; + } + + private executeModule(source: string, filename: string, isRunEntryPoint: boolean = false): any { + const ext = path.extname(filename).toLowerCase().replace(/^\./, ""); + if (ext === "json") { + const module = { + exports: JSON.parse(source), + id: filename, + filename, + loaded: true, + }; + this.moduleCache.set(filename, module); + return module.exports; + } + + let code = source; + if (this.compiler && this.sourceExtensions.has(ext)) { + code = this.compiler(code, filename); + } + + const fn = vm.compileFunction( + code, + ["exports", "require", "module", "__filename", "__dirname"], + { + filename, + parsingContext: this.context, + }, + ); + + const module = { + exports: {}, + id: filename, + filename, + loaded: false, + }; + this.moduleCache.set(filename, module); + + try { + const scopedRequire = this.createRequire(filename); + const result = fn.call( + module.exports, + module.exports, + scopedRequire, + module, + filename, + path.dirname(filename), + ); + module.loaded = true; + + return isRunEntryPoint && result !== undefined ? result : module.exports; + } catch (e) { + this.moduleCache.delete(filename); + throw e; + } + } + private getRealPath(targetPath: string): string { try { return fs.realpathSync(targetPath); @@ -384,7 +354,7 @@ export class VmRunner { if (isContainedIn(this.projectDir)) { return true; } - return this.allowedExternalPaths.some(allowed => isContainedIn(allowed)); + return this.allowedExternalPaths.some((allowed) => isContainedIn(allowed)); } private getStat(targetPath: string): fs.Stats | null { @@ -395,23 +365,33 @@ export class VmRunner { } } - private tryResolvePath(candidatePath: string): string | null { + private tryResolvePath( + candidatePath: string, + visitedDirs: Set = new Set(), + ): string | null { const stat = this.getStat(candidatePath); if (stat) { if (stat.isFile()) { return candidatePath; } if (stat.isDirectory()) { + if (visitedDirs.has(candidatePath)) { + return null; + } + visitedDirs.add(candidatePath); + const pkgPath = path.join(candidatePath, "package.json"); const pkgStat = this.getStat(pkgPath); if (pkgStat && pkgStat.isFile()) { try { const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); - if (pkg.main) { + if (pkg.main && typeof pkg.main === "string") { const mainPath = path.resolve(candidatePath, pkg.main); - const resolvedMain = this.tryResolvePath(mainPath); - if (resolvedMain) { - return resolvedMain; + if (mainPath !== candidatePath) { + const resolvedMain = this.tryResolvePath(mainPath, visitedDirs); + if (resolvedMain) { + return resolvedMain; + } } } } catch {} diff --git a/common/vm/vm_runner_test.ts b/common/vm/vm_runner_test.ts index ceb2defff..404f74196 100644 --- a/common/vm/vm_runner_test.ts +++ b/common/vm/vm_runner_test.ts @@ -27,7 +27,7 @@ suite("VmRunner", ({ afterEach }) => { const runner = new VmRunner({ projectDir: tmpDir }); const result = runner.run( JSON.stringify({ name: "dataform-test", active: true }), - path.join(tmpDir, "config.json") + path.join(tmpDir, "config.json"), ); expect(result).to.deep.equal({ name: "dataform-test", active: true }); }); @@ -38,7 +38,7 @@ suite("VmRunner", ({ afterEach }) => { fs.mkdirSync(path.join(tmpDir, "sub")); fs.writeFileSync( path.join(tmpDir, "sub", "helper.js"), - "module.exports = { greet: (x) => `Hello ${x}` };" + "module.exports = { greet: (x) => `Hello ${x}` };", ); const runner = new VmRunner({ projectDir: tmpDir }); @@ -55,7 +55,7 @@ suite("VmRunner", ({ afterEach }) => { fs.mkdirSync(path.join(tmpDir, "includes")); fs.writeFileSync( path.join(tmpDir, "includes", "math.js"), - "module.exports = { add: (a, b) => a + b };" + "module.exports = { add: (a, b) => a + b };", ); const runner = new VmRunner({ projectDir: tmpDir }); @@ -68,10 +68,7 @@ suite("VmRunner", ({ afterEach }) => { test("applies compiler hook to custom sourceExtensions", () => { const tmpDir = tmpDirFixture.createNewTmpDir(); - fs.writeFileSync( - path.join(tmpDir, "model.sqlx"), - "SELECT 1 AS id" - ); + fs.writeFileSync(path.join(tmpDir, "model.sqlx"), "SELECT 1 AS id"); const runner = new VmRunner({ projectDir: tmpDir, @@ -81,7 +78,7 @@ suite("VmRunner", ({ afterEach }) => { return `module.exports = { query: ${JSON.stringify(code.trim())}, file: ${JSON.stringify(filePath)} };`; } return code; - } + }, }); const result = runner.run(` @@ -100,7 +97,7 @@ suite("VmRunner", ({ afterEach }) => { exports.name = "moduleA"; const b = require("./b"); exports.getBName = () => b.name; - ` + `, ); fs.writeFileSync( path.join(tmpDir, "b.js"), @@ -108,7 +105,7 @@ suite("VmRunner", ({ afterEach }) => { exports.name = "moduleB"; const a = require("./a"); exports.getAName = () => a.name; - ` + `, ); const runner = new VmRunner({ projectDir: tmpDir }); @@ -125,7 +122,7 @@ suite("VmRunner", ({ afterEach }) => { const tmpDir = tmpDirFixture.createNewTmpDir(); const runner = new VmRunner({ projectDir: tmpDir, - builtinModules: ["path"] + builtinModules: ["path"], }); const pathResult = runner.run(` @@ -143,14 +140,14 @@ suite("VmRunner", ({ afterEach }) => { const tmpDir = tmpDirFixture.createNewTmpDir(); const mockCore = { version: "9.9.9", - compiler: () => "compiled" + compiler: () => "compiled", }; const runner = new VmRunner({ projectDir: tmpDir, mockModules: { - "@dataform/core": mockCore - } + "@dataform/core": mockCore, + }, }); const result = runner.run(` @@ -173,8 +170,8 @@ suite("VmRunner", ({ afterEach }) => { const runner = new VmRunner({ projectDir: tmpDir, sandbox: { - injectedValue: 123 - } + injectedValue: 123, + }, }); const result = runner.run(` @@ -210,12 +207,12 @@ suite("VmRunner", ({ afterEach }) => { // Relative path traversal const relativePath = path.relative(tmpDir, secretFile); expect(() => runner.run(`require(${JSON.stringify(relativePath)});`)).to.throw( - /outside of project directory/ + /outside of project directory/, ); // Absolute path traversal expect(() => runner.run(`require(${JSON.stringify(secretFile)});`)).to.throw( - /outside of project directory/ + /outside of project directory/, ); }); @@ -260,7 +257,7 @@ suite("VmRunner", ({ afterEach }) => { const runner = new VmRunner({ projectDir: tmpDir, - allowedExternalPaths: [sharedDir] + allowedExternalPaths: [sharedDir], }); const result = runner.run(` @@ -270,16 +267,23 @@ suite("VmRunner", ({ afterEach }) => { expect(result).to.equal("data"); }); - test("supports custom env and envAllowlist", () => { + test("supports custom env and envAllowlist, and defaults to empty env", () => { const tmpDir = tmpDirFixture.createNewTmpDir(); process.env.TEST_HOST_SECRET = "secret_123"; process.env.TEST_PUBLIC_VAR = "public_abc"; try { + // Default: empty environment to prevent leaking host secrets + const defaultRunner = new VmRunner({ projectDir: tmpDir }); + const defaultEnv = defaultRunner.run("return process.env;"); + expect(defaultEnv.TEST_HOST_SECRET).to.equal(undefined); + expect(defaultEnv.TEST_PUBLIC_VAR).to.equal(undefined); + expect(Object.keys(defaultEnv)).to.deep.equal([]); + // With envAllowlist const allowlistRunner = new VmRunner({ projectDir: tmpDir, - envAllowlist: ["TEST_PUBLIC_VAR"] + envAllowlist: ["TEST_PUBLIC_VAR"], }); const allowlistEnv = allowlistRunner.run("return process.env;"); expect(allowlistEnv.TEST_PUBLIC_VAR).to.equal("public_abc"); @@ -288,7 +292,7 @@ suite("VmRunner", ({ afterEach }) => { // With custom env record const customRunner = new VmRunner({ projectDir: tmpDir, - env: { CUSTOM_KEY: "custom_value" } + env: { CUSTOM_KEY: "custom_value" }, }); const customEnv = customRunner.run("return process.env;"); expect(customEnv.CUSTOM_KEY).to.equal("custom_value"); @@ -303,12 +307,9 @@ suite("VmRunner", ({ afterEach }) => { const tmpDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(tmpDir, "notebook.ipynb"), - JSON.stringify({ cells: [{ cell_type: "code", source: ["print('hello')"] }] }) - ); - fs.writeFileSync( - path.join(tmpDir, "doc.md"), - "# Hello Documentation" + JSON.stringify({ cells: [{ cell_type: "code", source: ["print('hello')"] }] }), ); + fs.writeFileSync(path.join(tmpDir, "doc.md"), "# Hello Documentation"); const runner = new VmRunner({ projectDir: tmpDir, @@ -321,7 +322,7 @@ suite("VmRunner", ({ afterEach }) => { return `module.exports = { asMarkdown: ${JSON.stringify(code)} };`; } return code; - } + }, }); const result = runner.run(` @@ -344,7 +345,7 @@ suite("VmRunner", ({ afterEach }) => { const runner = new VmRunner({ projectDir: tmpDir, sourceExtensions: ["sqlx"], - compiler: code => code + compiler: (code) => code, }); let caughtError: Error | null = null; @@ -381,10 +382,10 @@ suite("VmRunner", ({ afterEach }) => { compile: (bytes: Uint8Array) => { receivedBytes = bytes; return new Uint8Array([bytes[0] + 1, bytes[1] + 1]); - } - }) - } - } + }, + }), + }, + }, }); const result = runner.run(` @@ -401,5 +402,89 @@ suite("VmRunner", ({ afterEach }) => { expect(result.output).to.be.an.instanceOf(Uint8Array); expect(Array.from(result.output)).to.deep.equal([11, 21]); }); -}); + test("enforces allowedModules restriction when specified", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ + projectDir: tmpDir, + allowedModules: ["@dataform/*"], + mockModules: { + "@dataform/core": { name: "core" }, + "other-pkg": { name: "other" }, + }, + }); + + // Mocked modules are accessible + expect(runner.require("@dataform/core")).to.deep.equal({ name: "core" }); + + // Non-allowed non-mock module fails + let err: any = null; + try { + runner.require("unallowed-pkg"); + } catch (e) { + err = e; + } + expect(err).to.not.equal(null); + expect(err.message).to.include("Access to module 'unallowed-pkg' is not allowed"); + + // Project-relative internal files are still allowed even when allowedModules is specified + fs.mkdirSync(path.join(tmpDir, "includes")); + fs.writeFileSync(path.join(tmpDir, "includes", "helper.js"), "module.exports = { ok: true };"); + expect(runner.require("includes/helper")).to.deep.equal({ ok: true }); + }); + + test("does not get stuck in infinite recursion on self-referential package.json main", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const subDir = path.join(tmpDir, "loop_pkg"); + fs.mkdirSync(subDir); + fs.writeFileSync(path.join(subDir, "package.json"), JSON.stringify({ main: "." })); + fs.writeFileSync(path.join(subDir, "index.js"), "module.exports = { loaded: true };"); + + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.require("./loop_pkg", path.join(tmpDir, "index.js")); + expect(result).to.deep.equal({ loaded: true }); + }); + + test("re-throws unexpected errors from customResolve", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ + projectDir: tmpDir, + resolve: (moduleName) => { + if (moduleName === "fail-now") { + throw new TypeError("unexpected resolve error"); + } + return path.join(tmpDir, `${moduleName}.js`); + }, + }); + + let caught: any = null; + try { + runner.resolve("fail-now", path.join(tmpDir, "index.js")); + } catch (e) { + caught = e; + } + expect(caught).to.not.equal(null); + expect(caught).to.be.an.instanceOf(TypeError); + expect(caught.message).to.equal("unexpected resolve error"); + }); + + test("enforces isPathContained on node_modules symlinks pointing outside projectDir", () => { + const outsideDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(outsideDir, "external.js"), "module.exports = 'escaped';"); + + const projectDir = tmpDirFixture.createNewTmpDir(); + const nodeModulesDir = path.join(projectDir, "node_modules"); + fs.mkdirSync(nodeModulesDir); + fs.symlinkSync(outsideDir, path.join(nodeModulesDir, "symlinked-pkg")); + + const runner = new VmRunner({ projectDir }); + let caught: any = null; + try { + runner.resolve("symlinked-pkg/external", path.join(projectDir, "index.js")); + } catch (e) { + caught = e; + } + expect(caught).to.not.equal(null); + expect(caught.message).to.include("outside of project directory"); + }); +}); diff --git a/core/main_property_graphs_test.ts b/core/main_property_graphs_test.ts index 468b9ccb6..66ff26799 100644 --- a/core/main_property_graphs_test.ts +++ b/core/main_property_graphs_test.ts @@ -41,10 +41,15 @@ suite("property graphs", ({ afterEach }) => { message, ...extra, }); + // Under vm2, V8 stack traces were replaced with mocked "\n at CallSite {}". + // Under native node:vm, real V8 CallSites with genuine file paths and line numbers + // are preserved. We verify that compilation error stacks contain the error message + // and valid V8 call frames rather than vm2's mocked CallSite stubs. const asPlainGraph = (graph: dataform.ICompiledGraph) => { const plain = asPlainObject(graph); plain.graphErrors?.compilationErrors?.forEach((e: any) => { expect(e.stack).to.include(`Error: ${e.message}`); + expect(e.stack).to.match(/\n\s+at /); delete e.stack; }); return plain; diff --git a/readme.md b/readme.md index 9c23579c4..27e572dc3 100644 --- a/readme.md +++ b/readme.md @@ -27,6 +27,8 @@ You can run Dataform locally using the Dataform CLI tool, which can be installed npm i -g @dataform/cli ``` +> **Note on Project Execution & Security:** Dataform compiles SQLX and JavaScript project code using Node.js's native `node:vm` context. While module resolution is scoped to the project directory, `node:vm` is not a security sandbox against untrusted code. Dataform projects execute with the privileges of the local process; only run and compile Dataform projects from trusted sources. + ## Useful Links - [Documentation home page](https://cloud.google.com/dataform). diff --git a/testing/run_core.ts b/testing/run_core.ts index 9cd0e9a4d..3c5f79a13 100644 --- a/testing/run_core.ts +++ b/testing/run_core.ts @@ -108,12 +108,8 @@ export function runMainInVm( }, }); - const hasWorkflowSettingsYaml = fs.existsSync( - path.join(projectDir, "workflow_settings.yaml") - ); - const hasDataformJson = fs.existsSync( - path.join(projectDir, "dataform.json") - ); + const hasWorkflowSettingsYaml = fs.existsSync(path.join(projectDir, "workflow_settings.yaml")); + const hasDataformJson = fs.existsSync(path.join(projectDir, "dataform.json")); const encodedCoreExecutionRequest = encode64(dataform.CoreExecutionRequest, coreExecutionRequest); const vmIndexFileName = path.resolve(path.join(projectDir, "index.js")); @@ -123,8 +119,8 @@ export function runMainInVm( configurable: true, get: function() { return __df_current(); } }); - ${hasWorkflowSettingsYaml ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' : ''} - ${hasDataformJson ? 'global.dataformJson = require("./dataform.json");' : ''} + ${hasWorkflowSettingsYaml ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' : ""} + ${hasDataformJson ? 'global.dataformJson = require("./dataform.json");' : ""} return require("@dataform/core").main("${encodedCoreExecutionRequest}") `, vmIndexFileName, From 91477496e620dba8978c3573070bc95dcd32a604 Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Fri, 25 Sep 2026 18:33:42 +0000 Subject: [PATCH 4/4] Address review comments on VmRunner, part 2. --- cli/vm/BUILD | 22 +++++ cli/vm/compile.ts | 5 +- cli/vm/compile_test.ts | 152 +++++++++++++++++++++++++++++++ common/vm/vm_runner.ts | 73 ++++++++++----- common/vm/vm_runner_benchmark.ts | 8 +- common/vm/vm_runner_test.ts | 99 ++++++++++++++++++-- testing/run_core.ts | 9 +- 7 files changed, 335 insertions(+), 33 deletions(-) create mode 100644 cli/vm/compile_test.ts diff --git a/cli/vm/BUILD b/cli/vm/BUILD index a8fc54948..676940475 100644 --- a/cli/vm/BUILD +++ b/cli/vm/BUILD @@ -68,3 +68,25 @@ nodejs_binary( "--bazel_patch_module_resolver", ], ) + +ts_test_suite( + name = "tests", + srcs = [ + "compile_test.ts", + ], + data = [ + "//core:node_modules", + ], + deps = [ + ":vm", + "//common/protos", + "//protos:ts", + "//testing", + "@npm//@types/chai", + "@npm//@types/fs-extra", + "@npm//@types/node", + "@npm//chai", + "@npm//fs-extra", + ], +) + diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index 991719ec7..512453352 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -125,7 +125,10 @@ export function compile(compileConfig: dataform.ICompileConfig) { } ${ hasWorkflowSettingsYaml - ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' + ? `global.workflowSettingsYaml = (function() { + try { return require("./workflow_settings.yaml"); } + catch(e) { console.error("YAML require failed run_core:", e); } + })();` : "" } ${hasDataformJson ? 'global.dataformJson = require("./dataform.json");' : ""} diff --git a/cli/vm/compile_test.ts b/cli/vm/compile_test.ts new file mode 100644 index 000000000..9784720cb --- /dev/null +++ b/cli/vm/compile_test.ts @@ -0,0 +1,152 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; + +import { compile } from "df/cli/vm/compile"; +import { handleJitRequest } from "df/cli/vm/jit_worker"; +import { decode64 } from "df/common/protos"; +import { dataform } from "df/protos/ts"; +import { suite, test } from "df/testing"; +import { TmpDirFixture } from "df/testing/fixtures"; + +suite("cli/vm", ({ afterEach }) => { + const tmpDirFixture = new TmpDirFixture(afterEach); + + // Allow require("@dataform/core") to resolve to the prebuilt core bundle in the test environment + // tslint:disable-next-line: no-require-imports + const Module = require("module"); + const origResolve = Module._resolveFilename; + Module._resolveFilename = function (request: string, parent: any, isMain: boolean, options: any) { + if (request === "@dataform/core") { + return path.join(process.cwd(), "core", "node_modules", "@dataform", "core", "bundle.js"); + } + return origResolve.apply(this, arguments); + }; + + test("compile() runs end-to-end against prebuilt @dataform/core bundle", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + + // Copy built @dataform/core from Bazel runfiles into the project's node_modules. + fs.copySync( + path.join(process.cwd(), "core", "node_modules"), + path.join(projectDir, "node_modules"), + ); + + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: test-project +defaultDataset: test-dataset +defaultLocation: US +`, + ); + + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions", "example.sqlx"), + ` +config { + type: "table", + name: "example" +} +SELECT 1 AS col +`, + ); + fs.writeFileSync( + path.join(projectDir, "definitions", "actions.yaml"), + ` +actions: + - notebook: + filename: test_notebook.ipynb +`, + ); + fs.writeFileSync( + path.join(projectDir, "definitions", "test_notebook.ipynb"), + JSON.stringify({ cells: [] }), + ); + + const encodedResponse = compile({ projectDir }); + const response = decode64(dataform.CoreExecutionResponse, encodedResponse); + + expect(response.compile).to.be.an("object"); + expect(response.compile.compiledGraph).to.be.an("object"); + const tables = response.compile.compiledGraph.tables; + expect(tables).to.have.lengthOf(1); + expect(tables[0].target.name).to.equal("example"); + expect(tables[0].target.schema).to.equal("test-dataset"); + expect(tables[0].target.database).to.equal("test-project"); + + const notebooks = response.compile.compiledGraph.notebooks; + expect(notebooks).to.have.lengthOf(1); + expect(notebooks[0].target.name).to.equal("test_notebook"); + }); + + test("handleJitRequest compiles request with local core (hasProjectLocalCore = true)", async () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.copySync( + path.join(process.cwd(), "core", "node_modules"), + path.join(projectDir, "node_modules"), + ); + + const request = dataform.JitCompilationRequest.create({ + jitCode: `async (ctx) => "SELECT 1"`, + target: { + database: "db", + schema: "schema", + name: "test_op", + }, + compilationTargetType: + dataform.JitCompilationTargetType.JIT_COMPILATION_TARGET_TYPE_OPERATION, + }); + + const messages: any[] = []; + const origSend = process.send; + (process as any).send = (msg: any) => messages.push(msg); + + try { + await handleJitRequest({ request, projectDir }); + } finally { + (process as any).send = origSend; + } + + expect(messages).to.have.lengthOf(1); + expect(messages[0].type).to.equal("jit_response"); + expect(messages[0].response.operation.queries).to.deep.equal(["SELECT 1"]); + }); + + test("handleJitRequest compiles request with fallback core (hasProjectLocalCore = false)", async () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + // Provide only package.json for @dataform/core without bundle.js so hasProjectLocalCore is false + const coreDir = path.join(projectDir, "node_modules", "@dataform", "core"); + fs.mkdirSync(coreDir, { recursive: true }); + fs.writeFileSync( + path.join(coreDir, "package.json"), + JSON.stringify({ name: "@dataform/core", version: "3.0.0" }), + ); + + const request = dataform.JitCompilationRequest.create({ + jitCode: `async (ctx) => "SELECT 42"`, + target: { + database: "db", + schema: "schema", + name: "test_op2", + }, + compilationTargetType: + dataform.JitCompilationTargetType.JIT_COMPILATION_TARGET_TYPE_OPERATION, + }); + + const messages: any[] = []; + const origSend = process.send; + (process as any).send = (msg: any) => messages.push(msg); + + try { + await handleJitRequest({ request, projectDir }); + } finally { + (process as any).send = origSend; + } + + expect(messages).to.have.lengthOf(1); + expect(messages[0].type).to.equal("jit_response"); + expect(messages[0].response.operation.queries).to.deep.equal(["SELECT 42"]); + }); +}); diff --git a/common/vm/vm_runner.ts b/common/vm/vm_runner.ts index dbb76f202..f3abf5e0d 100644 --- a/common/vm/vm_runner.ts +++ b/common/vm/vm_runner.ts @@ -23,9 +23,21 @@ export interface VmRunnerOptions { sandbox?: Record; builtinModules?: string[]; mockModules?: Record; - resolve?: (moduleName: string, parentDirName: string) => string; + /** + * Optional custom resolver hook to resolve module names before falling back + * to standard project-relative or node_modules resolution. + */ + customResolve?: (moduleName: string, parentDirName: string) => string; console?: "inherit" | "off"; + /** + * Explicit map of environment variables exposed inside the VM context as `process.env`. + * Mutually exclusive with `envAllowlist`. + */ env?: Record; + /** + * Allowlist of environment variable names to copy from host `process.env` into the VM. + * Mutually exclusive with `env`. + */ envAllowlist?: string[]; allowedExternalPaths?: string[]; allowedModules?: string[]; @@ -73,9 +85,13 @@ export class VmRunner { options.builtinModules !== undefined ? options.builtinModules : ["path"], ); this.mockModules = options.mockModules || {}; - this.customResolve = options.resolve; + this.customResolve = options.customResolve; this.nodeBuiltinSet = new Set(nodeBuiltins); + if (options.env !== undefined && options.envAllowlist !== undefined) { + throw new Error("Cannot specify both 'env' and 'envAllowlist' in VmRunnerOptions"); + } + let env: Record; if (options.env !== undefined) { env = { ...options.env }; @@ -91,6 +107,11 @@ export class VmRunner { env = {}; } + const hrtime = process.hrtime.bind(process) as any; + if (typeof process.hrtime.bigint === "function") { + hrtime.bigint = process.hrtime.bigint.bind(process.hrtime); + } + const sandbox: Record = { console: options.console === "off" @@ -103,6 +124,8 @@ export class VmRunner { versions: process.versions, platform: process.platform, arch: process.arch, + hrtime, + nextTick: process.nextTick.bind(process), }, Buffer, Uint8Array, @@ -177,7 +200,9 @@ export class VmRunner { if (candidate) { const resolved = this.tryResolvePath(candidate); if (resolved) { - return this.checkContainmentAndCache(moduleName, resolved, cacheKey); + this.assertPathContained(resolved, moduleName); + this.resolveCache.set(cacheKey, resolved); + return resolved; } } } @@ -191,28 +216,34 @@ export class VmRunner { const candidate = path.resolve(parentDir, moduleName); const resolved = this.tryResolvePath(candidate); if (resolved) { - return this.checkContainmentAndCache(moduleName, resolved, cacheKey); + this.assertPathContained(resolved, moduleName); + this.resolveCache.set(cacheKey, resolved); + return resolved; } } else { - // Project-relative path (e.g. require("includes/helpers")) - const projectRelative = path.resolve(this.projectDir, moduleName); - const resolvedProjectRelative = this.tryResolvePath(projectRelative); - if (resolvedProjectRelative) { - return this.checkContainmentAndCache(moduleName, resolvedProjectRelative, cacheKey); - } - - // External module check: if allowedModules is specified, package must be allowed + // External module check: if allowedModules is specified, bare specifier must be allowed if (!this.isModuleAllowed(moduleName)) { const err: any = new Error(`Access to module '${moduleName}' is not allowed`); err.code = "MODULE_NOT_FOUND"; throw err; } + // Project-relative path (e.g. require("includes/helpers")) + const projectRelative = path.resolve(this.projectDir, moduleName); + const resolvedProjectRelative = this.tryResolvePath(projectRelative); + if (resolvedProjectRelative) { + this.assertPathContained(resolvedProjectRelative, moduleName); + this.resolveCache.set(cacheKey, resolvedProjectRelative); + return resolvedProjectRelative; + } + // Check project node_modules directory directly (e.g. @dataform/core) const nodeModulesCandidate = path.resolve(this.projectDir, "node_modules", moduleName); const resolvedNodeModules = this.tryResolvePath(nodeModulesCandidate); if (resolvedNodeModules) { - return this.checkContainmentAndCache(moduleName, resolvedNodeModules, cacheKey); + this.assertPathContained(resolvedNodeModules, moduleName); + this.resolveCache.set(cacheKey, resolvedNodeModules); + return resolvedNodeModules; } // Fallback to standard Node.js require.resolve resolution @@ -225,7 +256,9 @@ export class VmRunner { nodeReqError = e; } if (nodeReqResolved) { - return this.checkContainmentAndCache(moduleName, nodeReqResolved, cacheKey); + this.assertPathContained(nodeReqResolved, moduleName); + this.resolveCache.set(cacheKey, nodeReqResolved); + return nodeReqResolved; } let projectReqResolved: string | undefined; @@ -237,7 +270,9 @@ export class VmRunner { projectReqError = e; } if (projectReqResolved) { - return this.checkContainmentAndCache(moduleName, projectReqResolved, cacheKey); + this.assertPathContained(projectReqResolved, moduleName); + this.resolveCache.set(cacheKey, projectReqResolved); + return projectReqResolved; } const err: any = new Error(`Cannot find module '${moduleName}' from '${fromPath}'`); @@ -266,11 +301,7 @@ export class VmRunner { }); } - private checkContainmentAndCache( - moduleName: string, - resolvedPath: string, - cacheKey: string, - ): string { + private assertPathContained(resolvedPath: string, moduleName: string): void { if (!this.isPathContained(resolvedPath)) { const err: any = new Error( `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, @@ -278,8 +309,6 @@ export class VmRunner { err.code = "MODULE_NOT_FOUND"; throw err; } - this.resolveCache.set(cacheKey, resolvedPath); - return resolvedPath; } private executeModule(source: string, filename: string, isRunEntryPoint: boolean = false): any { diff --git a/common/vm/vm_runner_benchmark.ts b/common/vm/vm_runner_benchmark.ts index f2f444154..91f1567f5 100644 --- a/common/vm/vm_runner_benchmark.ts +++ b/common/vm/vm_runner_benchmark.ts @@ -9,14 +9,14 @@ function runBenchmark() { fs.mkdirSync(path.join(tmpDir, "includes")); fs.writeFileSync( path.join(tmpDir, "includes", "helpers.js"), - "module.exports = { format: (x) => 'formatted_' + x };" + "module.exports = { format: (x) => 'formatted_' + x };", ); for (let i = 0; i < 50; i++) { fs.writeFileSync( path.join(tmpDir, `table_${i}.js`), `const { format } = require("./includes/helpers"); - module.exports = { name: format("table_${i}"), query: "SELECT ${i}" };` + module.exports = { name: format("table_${i}"), query: "SELECT ${i}" };`, ); } @@ -37,7 +37,9 @@ function runBenchmark() { // eslint-disable-next-line no-console console.log("VmRunner Benchmark Results:"); // eslint-disable-next-line no-console - console.log(` Evaluated ${iterations} module requires in ${durationMs.toFixed(2)} ms (${opsPerSec} ops/sec)`); + console.log( + ` Evaluated ${iterations} module requires in ${durationMs.toFixed(2)} ms (${opsPerSec} ops/sec)`, + ); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } diff --git a/common/vm/vm_runner_test.ts b/common/vm/vm_runner_test.ts index 404f74196..cf39d533f 100644 --- a/common/vm/vm_runner_test.ts +++ b/common/vm/vm_runner_test.ts @@ -224,7 +224,7 @@ suite("VmRunner", ({ afterEach }) => { const runner = new VmRunner({ projectDir: tmpDir, - resolve: (moduleName) => path.resolve(outsideDir, moduleName), + customResolve: (moduleName: string) => path.resolve(outsideDir, moduleName), }); expect(() => runner.run(`require("secret.json");`)).to.throw(/outside of project directory/); @@ -427,10 +427,12 @@ suite("VmRunner", ({ afterEach }) => { expect(err).to.not.equal(null); expect(err.message).to.include("Access to module 'unallowed-pkg' is not allowed"); - // Project-relative internal files are still allowed even when allowedModules is specified - fs.mkdirSync(path.join(tmpDir, "includes")); - fs.writeFileSync(path.join(tmpDir, "includes", "helper.js"), "module.exports = { ok: true };"); - expect(runner.require("includes/helper")).to.deep.equal({ ok: true }); + // Bare specifier cannot bypass allowedModules even if a local file exists at project root + fs.writeFileSync(path.join(tmpDir, "lodash.js"), "module.exports = 'local-lodash';"); + expect(() => runner.require("lodash")).to.throw(/Access to module 'lodash' is not allowed/); + + // Relative require explicitly using ./ still accesses the local file + expect(runner.require("./lodash")).to.equal("local-lodash"); }); test("does not get stuck in infinite recursion on self-referential package.json main", () => { @@ -449,7 +451,7 @@ suite("VmRunner", ({ afterEach }) => { const tmpDir = tmpDirFixture.createNewTmpDir(); const runner = new VmRunner({ projectDir: tmpDir, - resolve: (moduleName) => { + customResolve: (moduleName: string) => { if (moduleName === "fail-now") { throw new TypeError("unexpected resolve error"); } @@ -487,4 +489,89 @@ suite("VmRunner", ({ afterEach }) => { expect(caught).to.not.equal(null); expect(caught.message).to.include("outside of project directory"); }); + + test("allows requiring node: prefixed builtin modules if allowed", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ + projectDir: tmpDir, + builtinModules: ["path"], + }); + + const result = runner.run(` + const path = require("node:path"); + return path.join("foo", "bar"); + `); + expect(result).to.equal(path.join("foo", "bar")); + }); + + test("suppresses console output when console is off", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ + projectDir: tmpDir, + console: "off", + }); + + expect(() => { + runner.run(` + console.log("hello"); + console.error("error"); + console.warn("warn"); + console.info("info"); + `); + }).not.to.throw(); + }); + + test("supports mocked modules being required and re-required from inside the VM", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const mockCore = { + version: "3.0.0", + compiler: () => "compiled", + }; + + const runner = new VmRunner({ + projectDir: tmpDir, + mockModules: { + "@dataform/core": mockCore, + }, + }); + + const result = runner.run(` + const core1 = require("@dataform/core"); + const core2 = require("@dataform/core"); + return { core1, same: core1 === core2 }; + `); + expect(result.core1.version).to.equal("3.0.0"); + expect(result.same).to.equal(true); + }); + + test("throws when both env and envAllowlist are specified", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + expect(() => { + new VmRunner({ + projectDir: tmpDir, + env: { TEST: "1" }, + envAllowlist: ["TEST"], + }); + }).to.throw("Cannot specify both 'env' and 'envAllowlist' in VmRunnerOptions"); + }); + + test("provides process.hrtime and process.nextTick in the process shim", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ projectDir: tmpDir }); + + const result = runner.run(` + const [seconds, nanos] = process.hrtime(); + const bigintTime = typeof process.hrtime.bigint === "function" ? process.hrtime.bigint() : 0n; + let nextTickCalled = false; + process.nextTick(() => { nextTickCalled = true; }); + return { + hasHrtime: typeof seconds === "number" && typeof nanos === "number", + hasBigint: typeof bigintTime === "bigint", + hasNextTick: typeof process.nextTick === "function", + }; + `); + expect(result.hasHrtime).to.equal(true); + expect(result.hasBigint).to.equal(true); + expect(result.hasNextTick).to.equal(true); + }); }); diff --git a/testing/run_core.ts b/testing/run_core.ts index 3c5f79a13..9f305d3a8 100644 --- a/testing/run_core.ts +++ b/testing/run_core.ts @@ -119,7 +119,14 @@ export function runMainInVm( configurable: true, get: function() { return __df_current(); } }); - ${hasWorkflowSettingsYaml ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' : ""} + ${ + hasWorkflowSettingsYaml + ? `global.workflowSettingsYaml = (function() { + try { return require("./workflow_settings.yaml"); } + catch(e) { console.error("YAML require failed run_core:", e); } + })();` + : "" + } ${hasDataformJson ? 'global.dataformJson = require("./dataform.json");' : ""} return require("@dataform/core").main("${encodedCoreExecutionRequest}") `,