diff --git a/cli/api/BUILD b/cli/api/BUILD index 9b817c962..fa015c14e 100644 --- a/cli/api/BUILD +++ b/cli/api/BUILD @@ -41,6 +41,7 @@ ts_library( "@npm//fs-extra", "@npm//glob", "@npm//google-sql-syntax-ts", + "@npm//ignore", "@npm//js-beautify", "@npm//js-yaml", "@npm//promise-pool-executor", @@ -62,6 +63,7 @@ ts_test_suite( srcs = [ "tasks_test.ts", "utils_test.ts", + "commands/compile_copy_filter_test.ts", "commands/jit/rpc_test.ts", "commands/prune_test.ts", "dbadapters/bigquery_test.ts", diff --git a/cli/api/commands/compile.ts b/cli/api/commands/compile.ts index 8ef7c78c8..5a193afd2 100644 --- a/cli/api/commands/compile.ts +++ b/cli/api/commands/compile.ts @@ -5,6 +5,10 @@ import * as tmp from "tmp"; import { promisify } from "util"; import { BaseWorker } from "df/cli/api/commands/base_worker"; +import { + buildProjectCopyFilter, + findProjectIgnoreFiles, +} from "df/cli/api/commands/compile_copy_filter"; import { MISSING_CORE_VERSION_ERROR } from "df/cli/api/commands/install"; import { readConfigFromWorkflowSettings } from "df/cli/api/utils"; import { DEFAULT_COMPILATION_TIMEOUT_MILLIS } from "df/cli/api/utils/constants"; @@ -54,9 +58,17 @@ export async function compile( `Using isolated environment for @dataform/core@${workflowSettingsDataformCoreVersion}\n`, ); print(`Copying project to temporary directory: ${temporaryProjectPath}\n`); + const ignoreFiles = findProjectIgnoreFiles(resolvedProjectPath); + print( + ignoreFiles.length > 0 + ? `Excluding .git, node_modules, and paths matched by: ${ignoreFiles.join(", ")}\n` + : `Excluding .git and node_modules (no .gitignore or .dataformignore in project root)\n`, + ); } const copyStartTime = performance.now(); - fs.copySync(resolvedProjectPath, temporaryProjectPath); + fs.copySync(resolvedProjectPath, temporaryProjectPath, { + filter: buildProjectCopyFilter(resolvedProjectPath), + }); if (compileConfig.verbose) { print(`Project copy completed in ${performance.now() - copyStartTime}ms\n`); } diff --git a/cli/api/commands/compile_copy_filter.ts b/cli/api/commands/compile_copy_filter.ts new file mode 100644 index 000000000..e31825ac4 --- /dev/null +++ b/cli/api/commands/compile_copy_filter.ts @@ -0,0 +1,146 @@ +import * as fs from "fs-extra"; +import ignore from "ignore"; +import * as path from "path"; + +// Excluded whatever the project's ignore files say. `.git` holds no Dataform project +// files. A top-level `node_modules` can't be present at all here -- `compile()` rejects +// the project before copying if it finds one -- so that entry covers nested ones, which +// are likewise never part of a Dataform project. +// +// Checked independently of the `ignore` instance below, rather than seeded into it, so +// that a project's ignore files cannot override this floor: `ignore` lets later patterns +// override earlier ones by design, so a `!node_modules` negation would otherwise +// un-ignore it. +const ALWAYS_IGNORED_NAMES = new Set([".git", "node_modules"]); + +// Project-root files that are always copied, whatever the project's ignore files say. +// `compile()` has already read `workflow_settings.yaml` from the original project to +// decide on a stateless install, and compilation in the copy can't proceed without it, +// so a broad pattern like `*.yaml` must not drop it. +const ALWAYS_COPIED_ROOT_FILES = new Set(["workflow_settings.yaml"]); + +// Ignore files read from the project root, in this order. Later patterns override earlier +// ones, so a `!pattern` in `.dataformignore` can un-ignore a path the `.gitignore` excludes +// (for example, definitions generated into a gitignored directory). +export const PROJECT_IGNORE_FILE_NAMES = [".gitignore", ".dataformignore"]; + +/** + * Returns the names of the PROJECT_IGNORE_FILE_NAMES present in the project root as + * files, in the order they are applied. Anything else by that name, such as a directory, + * is not an ignore file and is skipped rather than failing the compile. + */ +export function findProjectIgnoreFiles(resolvedProjectPath: string): string[] { + return PROJECT_IGNORE_FILE_NAMES.filter((name) => { + const ignoreFilePath = path.join(resolvedProjectPath, name); + return fs.existsSync(ignoreFilePath) && fs.statSync(ignoreFilePath).isFile(); + }); +} + +function swapCase(name: string): string { + return name + .split("") + .map((c) => (c === c.toUpperCase() ? c.toLowerCase() : c.toUpperCase())) + .join(""); +} + +/** + * Returns whether names in `directory` are matched case-insensitively by its filesystem, + * as on default Windows and macOS volumes. It probes an existing entry under its + * case-swapped name: if that resolves but isn't itself listed, it's the same entry. + * Falls back to the platform default when no entry has a name with letters. + */ +export function isCaseInsensitiveDirectory(directory: string): boolean { + const entries = fs.readdirSync(directory); + for (const entry of entries) { + const swapped = swapCase(entry); + if (swapped !== entry) { + return !entries.includes(swapped) && fs.existsSync(path.join(directory, swapped)); + } + } + return process.platform === "win32" || process.platform === "darwin"; +} + +/** + * Builds a filter for fs-extra's `copySync`, so the stateless-install copy in `compile()` + * skips files that can't be part of the Dataform project -- most commonly a large + * `.venv`, build-output or cache directory sitting alongside `definitions/`, whose size + * the copy would otherwise pay for. + * + * Exclusions come from the project's own `.gitignore` rather than from a hardcoded list + * of directory names: no fixed list covers every ecosystem's junk directories (`.venv`, + * `target/`, `__pycache__/`, `vendor/`, `coverage/`, ...), whereas a project's + * `.gitignore` already states exactly what that project treats as disposable, and + * `dataform init` writes one. An optional `.dataformignore`, in the same syntax, is + * applied on top of it: it can exclude further paths, or un-ignore gitignored ones with + * `!pattern`. + * + * Only ignore files in the project root are read. Nested `.gitignore` files, + * `.git/info/exclude` and the user's global excludes file are not consulted, so a + * project relying on those has more copied than `git status` would suggest. A project + * with neither file gets only the ALWAYS_IGNORED_NAMES floor. + * + * Note that an ignored file is never copied, so it is also never compiled: a project + * that generates definitions into a gitignored path needs that path unignored, in + * either file. As in git, a file can't be re-included while any ancestor directory is + * still excluded -- the copy never descends into that directory -- so each excluded + * ancestor must be un-ignored too. With `definitions/generated/` in `.gitignore`, + * `!definitions/generated/gen.sqlx` alone has no effect; `!definitions/generated/` + * restores the directory. + * + * Names are matched with the case sensitivity of the project's filesystem, as git does + * with `core.ignorecase`: on a case-sensitive one, `definitions/staging/` doesn't match + * `definitions/Staging/`, and `NODE_MODULES` is an ordinary directory; on a + * case-insensitive one, both patterns and negations match regardless of case. That + * applies to the ALWAYS_IGNORED_NAMES and ALWAYS_COPIED_ROOT_FILES checks too. + * `caseInsensitive` is detected from the project directory unless given. + * + * Patterns are evaluated on their own, without consulting git's index, so a file git + * still tracks despite matching a pattern (for example, one force-added with + * `git add -f`) is excluded all the same. The exception is ALWAYS_COPIED_ROOT_FILES. + */ +export function buildProjectCopyFilter( + resolvedProjectPath: string, + caseInsensitive = isCaseInsensitiveDirectory(resolvedProjectPath), +): (src: string) => boolean { + const normalizeCase = (name: string) => (caseInsensitive ? name.toLowerCase() : name); + const ig = ignore({ ignorecase: caseInsensitive }); + for (const name of findProjectIgnoreFiles(resolvedProjectPath)) { + ig.add(fs.readFileSync(path.join(resolvedProjectPath, name), "utf8")); + } + + return (src: string) => { + const relative = path.relative(resolvedProjectPath, src); + // The project root itself (relative === ""), or something outside the project + // root (shouldn't happen in practice for a copySync(resolvedProjectPath, ...) + // call, but not this function's place to decide) is always copied/recursed into. + if ( + !relative || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + return true; + } + + if (ALWAYS_COPIED_ROOT_FILES.has(normalizeCase(relative))) { + return true; + } + + const relativeSegments = relative.split(path.sep); + if (relativeSegments.some((segment) => ALWAYS_IGNORED_NAMES.has(normalizeCase(segment)))) { + return false; + } + + // `ignore` needs to know whether a path is a directory to correctly match + // patterns like `.venv/` (trailing slash = directories only), and fs-extra's + // copySync filter callback isn't given that -- only `src`. Use lstatSync so + // dangling symlinks remain copyable, matching copySync's default behavior of + // copying links rather than dereferencing them. + let posixRelative = relativeSegments.join("/"); + if (fs.lstatSync(src).isDirectory()) { + posixRelative += "/"; + } + + return !ig.ignores(posixRelative); + }; +} diff --git a/cli/api/commands/compile_copy_filter_test.ts b/cli/api/commands/compile_copy_filter_test.ts new file mode 100644 index 000000000..3dafcacb5 --- /dev/null +++ b/cli/api/commands/compile_copy_filter_test.ts @@ -0,0 +1,325 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; + +import { + buildProjectCopyFilter, + findProjectIgnoreFiles, + isCaseInsensitiveDirectory, +} from "df/cli/api/commands/compile_copy_filter"; +import { suite, test } from "df/testing"; +import { TmpDirFixture } from "df/testing/fixtures"; + +suite("buildProjectCopyFilter", ({ afterEach }) => { + const tmpDirFixture = new TmpDirFixture(afterEach); + + test("with no .gitignore, only .git and node_modules are excluded", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.ensureDirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions", "foo.sqlx"), "SELECT 1"); + fs.ensureDirSync(path.join(projectDir, ".venv")); + fs.ensureDirSync(path.join(projectDir, ".git")); + fs.ensureDirSync(path.join(projectDir, "node_modules")); + fs.ensureDirSync(path.join(projectDir, "definitions", "nested", "node_modules")); + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(projectDir)).to.equal(true); + expect(filter(path.join(projectDir, "definitions"))).to.equal(true); + expect(filter(path.join(projectDir, "definitions", "foo.sqlx"))).to.equal(true); + expect(filter(path.join(projectDir, ".venv"))).to.equal(true); + + expect(filter(path.join(projectDir, ".git"))).to.equal(false); + expect(filter(path.join(projectDir, "node_modules"))).to.equal(false); + // Excluded at any depth, not just at the project root. + expect(filter(path.join(projectDir, "definitions", "nested", "node_modules"))).to.equal(false); + }); + + test("does not dereference symlinks while filtering", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const danglingSymlink = path.join(projectDir, "dangling-link"); + fs.symlinkSync(path.join(projectDir, "missing-target"), danglingSymlink); + + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(danglingSymlink)).to.equal(true); + }); + + test("applies ignore rules to in-project paths beginning with two dots", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const ignoredDir = path.join(projectDir, "..cache"); + fs.ensureDirSync(ignoredDir); + fs.writeFileSync(path.join(projectDir, ".gitignore"), "..cache/\n"); + + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(ignoredDir)).to.equal(false); + }); + + test("the always-ignored floor cannot be overridden by a negation pattern", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.ensureDirSync(path.join(projectDir, "node_modules")); + // A project .gitignore is user-controlled and could (unusually, but validly) + // contain a negation pattern for something we always want to exclude. + fs.writeFileSync(path.join(projectDir, ".gitignore"), "!node_modules\n"); + + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(path.join(projectDir, "node_modules"))).to.equal(false); + }); + + test("filters an actual project copy", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const destinationDir = tmpDirFixture.createNewTmpDir(); + fs.ensureDirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions", "foo.sqlx"), "SELECT 1"); + fs.ensureDirSync(path.join(projectDir, ".venv")); + fs.writeFileSync(path.join(projectDir, ".venv", "ignored"), "junk"); + fs.ensureDirSync(path.join(projectDir, "node_modules")); + fs.writeFileSync(path.join(projectDir, "node_modules", "ignored"), "junk"); + fs.writeFileSync(path.join(projectDir, ".gitignore"), ".venv/\n"); + + fs.copySync(projectDir, destinationDir, { + filter: buildProjectCopyFilter(projectDir), + }); + + expect(fs.readFileSync(path.join(destinationDir, "definitions", "foo.sqlx"), "utf8")).to.equal( + "SELECT 1", + ); + expect(fs.existsSync(path.join(destinationDir, ".venv"))).to.equal(false); + expect(fs.existsSync(path.join(destinationDir, "node_modules"))).to.equal(false); + }); + + test("respects a project .gitignore, in addition to the always-ignored floor", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, ".gitignore"), + [".venv/", "__pycache__/", "*.pyc"].join("\n"), + ); + fs.ensureDirSync(path.join(projectDir, ".venv", "lib")); + fs.writeFileSync(path.join(projectDir, ".venv", "lib", "mod.py"), "# stub"); + fs.ensureDirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions", "foo.sqlx"), "SELECT 1"); + fs.writeFileSync(path.join(projectDir, "foo.pyc"), "junk"); + fs.ensureDirSync(path.join(projectDir, ".git")); + fs.ensureDirSync(path.join(projectDir, "node_modules")); + + const filter = buildProjectCopyFilter(projectDir); + + // Dataform-relevant paths are still copied. + expect(filter(projectDir)).to.equal(true); + expect(filter(path.join(projectDir, "definitions"))).to.equal(true); + expect(filter(path.join(projectDir, "definitions", "foo.sqlx"))).to.equal(true); + expect(filter(path.join(projectDir, ".gitignore"))).to.equal(true); + + // gitignore'd paths are excluded -- including the bare directory itself (not + // just its contents), which requires correctly detecting it as a directory to + // match a trailing-slash-only pattern like `.venv/`. + expect(filter(path.join(projectDir, ".venv"))).to.equal(false); + expect(filter(path.join(projectDir, ".venv", "lib", "mod.py"))).to.equal(false); + expect(filter(path.join(projectDir, "foo.pyc"))).to.equal(false); + + // The always-ignored floor still applies even when a .gitignore is present. + expect(filter(path.join(projectDir, ".git"))).to.equal(false); + expect(filter(path.join(projectDir, "node_modules"))).to.equal(false); + }); + + test("a .dataformignore excludes paths on its own, with no .gitignore", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(projectDir, ".dataformignore"), "scratch/\n"); + fs.ensureDirSync(path.join(projectDir, "scratch")); + fs.ensureDirSync(path.join(projectDir, "definitions")); + + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(path.join(projectDir, "scratch"))).to.equal(false); + expect(filter(path.join(projectDir, "definitions"))).to.equal(true); + }); + + test("a .dataformignore supplements the .gitignore and can un-ignore its paths", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const destinationDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, ".gitignore"), + [".venv/", "definitions/generated/"].join("\n"), + ); + fs.writeFileSync( + path.join(projectDir, ".dataformignore"), + ["docs/", "!definitions/generated/"].join("\n"), + ); + fs.ensureDirSync(path.join(projectDir, ".venv")); + fs.writeFileSync(path.join(projectDir, ".venv", "ignored"), "junk"); + fs.ensureDirSync(path.join(projectDir, "docs")); + fs.writeFileSync(path.join(projectDir, "docs", "ignored.md"), "junk"); + fs.ensureDirSync(path.join(projectDir, "definitions", "generated")); + fs.writeFileSync(path.join(projectDir, "definitions", "generated", "gen.sqlx"), "SELECT 1"); + + fs.copySync(projectDir, destinationDir, { + filter: buildProjectCopyFilter(projectDir), + }); + + // Still excluded by the .gitignore. + expect(fs.existsSync(path.join(destinationDir, ".venv"))).to.equal(false); + // Excluded by the .dataformignore. + expect(fs.existsSync(path.join(destinationDir, "docs"))).to.equal(false); + // Gitignored, but un-ignored by the .dataformignore. + expect( + fs.readFileSync(path.join(destinationDir, "definitions", "generated", "gen.sqlx"), "utf8"), + ).to.equal("SELECT 1"); + }); + + test("a .dataformignore cannot override the always-ignored floor", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.ensureDirSync(path.join(projectDir, "node_modules")); + fs.writeFileSync(path.join(projectDir, ".dataformignore"), "!node_modules\n"); + + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(path.join(projectDir, "node_modules"))).to.equal(false); + }); + + test("findProjectIgnoreFiles lists the ignore files present, in application order", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + expect(findProjectIgnoreFiles(projectDir)).to.deep.equal([]); + + fs.writeFileSync(path.join(projectDir, ".dataformignore"), ""); + expect(findProjectIgnoreFiles(projectDir)).to.deep.equal([".dataformignore"]); + + fs.writeFileSync(path.join(projectDir, ".gitignore"), ""); + expect(findProjectIgnoreFiles(projectDir)).to.deep.equal([".gitignore", ".dataformignore"]); + }); + + test("on a case-sensitive filesystem, patterns only match their exact case", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const destinationDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(projectDir, ".gitignore"), "definitions/staging/\n"); + fs.ensureDirSync(path.join(projectDir, "definitions", "Staging")); + fs.writeFileSync(path.join(projectDir, "definitions", "Staging", "table.sqlx"), "SELECT 1"); + + fs.copySync(projectDir, destinationDir, { + filter: buildProjectCopyFilter(projectDir, false), + }); + + expect( + fs.readFileSync(path.join(destinationDir, "definitions", "Staging", "table.sqlx"), "utf8"), + ).to.equal("SELECT 1"); + }); + + test("un-ignoring only a file does not re-include it while its directory is ignored", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const destinationDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(projectDir, ".gitignore"), "definitions/generated/\n"); + fs.writeFileSync(path.join(projectDir, ".dataformignore"), "!definitions/generated/gen.sqlx\n"); + fs.ensureDirSync(path.join(projectDir, "definitions", "generated")); + fs.writeFileSync(path.join(projectDir, "definitions", "generated", "gen.sqlx"), "SELECT 1"); + + fs.copySync(projectDir, destinationDir, { + filter: buildProjectCopyFilter(projectDir), + }); + + // Matches git: the excluded parent directory must itself be un-ignored. + expect(fs.existsSync(path.join(destinationDir, "definitions", "generated"))).to.equal(false); + }); + + test("on a case-insensitive filesystem, patterns and negations ignore case", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(projectDir, ".gitignore"), "definitions/staging/\n*.sqlx\n"); + fs.writeFileSync(path.join(projectDir, ".dataformignore"), "!definitions/Keep.sqlx\n"); + fs.ensureDirSync(path.join(projectDir, "definitions", "Staging")); + fs.writeFileSync(path.join(projectDir, "definitions", "keep.sqlx"), "SELECT 1"); + fs.writeFileSync(path.join(projectDir, "definitions", "drop.sqlx"), "SELECT 1"); + + const filter = buildProjectCopyFilter(projectDir, true); + + expect(filter(path.join(projectDir, "definitions", "Staging"))).to.equal(false); + expect(filter(path.join(projectDir, "definitions", "keep.sqlx"))).to.equal(true); + expect(filter(path.join(projectDir, "definitions", "drop.sqlx"))).to.equal(false); + }); + + test("on a case-sensitive filesystem, negations only match their exact case", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(projectDir, ".gitignore"), "*.sqlx\n!definitions/Keep.sqlx\n"); + fs.ensureDirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions", "keep.sqlx"), "SELECT 1"); + fs.writeFileSync(path.join(projectDir, "definitions", "Keep.sqlx"), "SELECT 1"); + + const filter = buildProjectCopyFilter(projectDir, false); + + expect(filter(path.join(projectDir, "definitions", "Keep.sqlx"))).to.equal(true); + expect(filter(path.join(projectDir, "definitions", "keep.sqlx"))).to.equal(false); + }); + + test("the always-ignored floor follows the filesystem's case sensitivity", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.ensureDirSync(path.join(projectDir, ".GIT")); + fs.ensureDirSync(path.join(projectDir, "definitions", "NODE_MODULES")); + fs.writeFileSync(path.join(projectDir, "definitions", "NODE_MODULES", "table.sqlx"), ""); + fs.writeFileSync(path.join(projectDir, ".gitignore"), "!.GIT\n!NODE_MODULES\n"); + + // Case-insensitive: these are the excluded directories, negations notwithstanding. + const insensitiveFilter = buildProjectCopyFilter(projectDir, true); + expect(insensitiveFilter(path.join(projectDir, ".GIT"))).to.equal(false); + expect(insensitiveFilter(path.join(projectDir, "definitions", "NODE_MODULES"))).to.equal(false); + + // Case-sensitive: these are ordinary directories. + const sensitiveFilter = buildProjectCopyFilter(projectDir, false); + expect(sensitiveFilter(path.join(projectDir, ".GIT"))).to.equal(true); + expect( + sensitiveFilter(path.join(projectDir, "definitions", "NODE_MODULES", "table.sqlx")), + ).to.equal(true); + }); + + test("workflow_settings.yaml is always copied, even when an ignore file matches it", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const destinationDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(projectDir, ".gitignore"), "*.yaml\n"); + fs.writeFileSync(path.join(projectDir, "workflow_settings.yaml"), "defaultProject: p\n"); + fs.writeFileSync(path.join(projectDir, "other.yaml"), "junk"); + + fs.copySync(projectDir, destinationDir, { + filter: buildProjectCopyFilter(projectDir), + }); + + expect(fs.readFileSync(path.join(destinationDir, "workflow_settings.yaml"), "utf8")).to.equal( + "defaultProject: p\n", + ); + expect(fs.existsSync(path.join(destinationDir, "other.yaml"))).to.equal(false); + }); + + test("an ignore file name that is a directory is skipped, not read", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.ensureDirSync(path.join(projectDir, ".dataformignore")); + fs.writeFileSync(path.join(projectDir, ".gitignore"), ".venv/\n"); + fs.ensureDirSync(path.join(projectDir, ".venv")); + + expect(findProjectIgnoreFiles(projectDir)).to.deep.equal([".gitignore"]); + const filter = buildProjectCopyFilter(projectDir); + expect(filter(path.join(projectDir, ".venv"))).to.equal(false); + }); + + test("on a case-insensitive filesystem, workflow_settings.yaml is kept in any case", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(projectDir, ".gitignore"), "*.yaml\n"); + fs.writeFileSync(path.join(projectDir, "Workflow_Settings.yaml"), "defaultProject: p\n"); + + expect( + buildProjectCopyFilter(projectDir, true)(path.join(projectDir, "Workflow_Settings.yaml")), + ).to.equal(true); + expect( + buildProjectCopyFilter(projectDir, false)(path.join(projectDir, "Workflow_Settings.yaml")), + ).to.equal(false); + }); + + test("isCaseInsensitiveDirectory agrees with how the filesystem resolves names", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(projectDir, "probe"), ""); + const expected = fs.existsSync(path.join(projectDir, "PROBE")); + + expect(isCaseInsensitiveDirectory(projectDir)).to.equal(expected); + + // With both case variants present as distinct entries, it must be case-sensitive. + if (!expected) { + fs.writeFileSync(path.join(projectDir, "PROBE"), ""); + expect(isCaseInsensitiveDirectory(projectDir)).to.equal(false); + } + }); +}); diff --git a/package.json b/package.json index 1000b614b..aba30509c 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "fs-extra": "^9.0.0", "glob": "13.0.6", "google-sql-syntax-ts": "^1.0.3", + "ignore": "^5.2.0", "js-beautify": "1.15.4", "js-yaml": "^4.3.2", "long": "^4.0.0", diff --git a/packages/@dataform/cli/BUILD b/packages/@dataform/cli/BUILD index 31309c581..b625504f0 100644 --- a/packages/@dataform/cli/BUILD +++ b/packages/@dataform/cli/BUILD @@ -38,6 +38,7 @@ externals = [ "fs-extra", "glob", "google-sql-syntax-ts", + "ignore", "js-beautify", "js-yaml", "moo",