diff --git a/src/lib/find-actor-dirs.ts b/src/lib/find-actor-dirs.ts index d945210ea..6fdd6550d 100644 --- a/src/lib/find-actor-dirs.ts +++ b/src/lib/find-actor-dirs.ts @@ -73,7 +73,10 @@ export async function findActorDirs(root: string): Promise { return []; } - const isGitignored = await getGitignoreMatcher(root); + const isGitignored = await getGitignoreMatcher( + root, + found.map((actor) => `${actor.dir}/`), + ); return found .map((actor) => ({ ...actor, ignored: isGitignored(`${actor.dir}/`) })) diff --git a/src/lib/utils.ts b/src/lib/utils.ts index f4dcecaab..e055dbcf7 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,8 +1,8 @@ import { execSync } from 'node:child_process'; -import { createWriteStream, existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createWriteStream, existsSync, readdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; import { mkdir, readFile } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { dirname, join, relative, resolve } from 'node:path'; +import { dirname, join, relative, resolve, sep } from 'node:path'; import process from 'node:process'; import { DurationFormatter as SapphireDurationFormatter, TimeTypes } from '@sapphire/duration'; @@ -289,99 +289,178 @@ export const createSourceFiles = async (paths: string[], cwd: string) => { }; /** - * Fallback for when git is unavailable: find all .gitignore files and build a matcher - * using the `ignore` package, scoped to each file's directory. - * Also walks ancestor directories to pick up parent .gitignore files (e.g. monorepo root), - * stopping at the first .git boundary found. + * Output of a git command run in `cwd`, or `null` when git fails (not installed, not a repository, refuses to run) or + * exits with an error code. */ -const getGitignoreFallbackMatcher = async (cwd: string): Promise<(path: string) => boolean> => { +const gitOutput = (args: string, cwd: string, input?: string): string | null => { + try { + return execSync(`git ${args}`, { + cwd, + input, + encoding: 'utf-8', + stdio: [input === undefined ? 'ignore' : 'pipe', 'pipe', 'ignore'], + }); + } catch { + return null; + } +}; + +const readFileOrEmpty = async (path: string) => readFile(path, 'utf-8').catch(() => ''); + +/** `to` relative to `from`, with `/` separators, as ignore rules and the paths they match use. */ +const posixRelative = (from: string, to: string) => relative(from, to).split(sep).join('/'); + +/** + * Lines of an ignore file in a directory above `cwd`, where `cwdPath` leads from that directory to `cwd`. When they + * ignore `cwd` or a directory between, the lines that do are dropped, so the files inside still get the other rules. + * Pushing an ignored directory is explicit, as `apify push --dir` is. + */ +const keepLinesForCwd = (content: string, cwdPath: string): string[] => { + // In NFC, as the final matcher compares them, so a line in another Unicode form than the path still counts. + const lines = content.normalize('NFC').split(/\r?\n/); + const cwdDir = `${cwdPath}/`.normalize('NFC'); + if (!cwdPath || !makeIg().add(lines).ignores(cwdDir)) { + return lines; + } + + return lines.filter((line) => !makeIg().add(line).ignores(cwdDir)); +}; + +/** + * Rewrites the lines of an ignore file in the directory `dir` to match paths relative to the directory `dir` is relative + * to, as git scopes them: a pattern with a slash before its end is anchored to `dir`, one without matches at any depth + * below it. + */ +const scopeLines = (lines: string[], dir: string): string[] => { + if (!dir) { + return lines; + } + + // The directory starts every rewritten pattern, so its glob characters, and a leading ! or #, are escaped. + const base = dir.replace(/[\\*?[\]]/g, '\\$&').replace(/^[!#]/, '\\$&'); + return lines.flatMap((line) => { + // Git drops trailing spaces unless a backslash escapes them. + const rule = line.replace(/(? { + const infoExclude = gitOutput('rev-parse --git-path info/exclude', cwd)?.trim(); + if (!infoExclude) { + return []; + } + + // An empty core.excludesFile turns the global excludes file off. + const globalExcludes = + gitOutput('config --path --get core.excludesFile', cwd)?.trim() ?? + join(process.env.XDG_CONFIG_HOME || join(homedir(), '.config'), 'git', 'ignore'); + + return [...(globalExcludes ? [resolve(root, globalExcludes)] : []), resolve(cwd, infoExclude)]; +}; + +/** + * Reads the ignore rules itself, for when git can't answer: git fails, or `cwd` is itself ignored. Applies every + * .gitignore inside `cwd` and every one above it up to and including the repository root, the first directory with a + * .git, so an outer repository's rules don't apply. When git names them, also applies .git/info/exclude and the global + * excludes file, as `git ls-files --exclude-standard` does. In the files above `cwd`, the lines that ignore `cwd` are + * dropped. + * + * The rules rank as git ranks them. A deeper .gitignore wins over a shallower one, so its `!` lines re-include files. + * Every .gitignore wins over .git/info/exclude, which wins over the global excludes file. A file below an ignored + * directory stays ignored. + */ +const getGitignoreRulesMatcher = async (typedCwd: string): Promise<(path: string) => boolean> => { + // Git applies the rules by the real path, so a directory pushed through a symlink gets the ancestors and the + // .git/info/exclude path git uses. A directory that doesn't exist holds no files, and the push says so. + const cwd = existsSync(typedCwd) ? realpathSync(typedCwd) : typedCwd; + + // Directories above cwd from the top down: the repository root, or the filesystem root when there's none. + const ancestors: string[] = []; + let root = cwd; + while (!existsSync(join(root, '.git')) && root !== dirname(root)) { + root = dirname(root); + ancestors.unshift(root); + } + const gitignoreFiles = await glob('**/.gitignore', { dot: true, cwd, ignore: ['.git/**'], expandDirectories: false, }); + gitignoreFiles.sort((a, b) => a.split('/').length - b.split('/').length); - const filters: { dir: string; ig: Ignore; ancestorPrefix?: string }[] = []; - - for (const gitignoreFile of gitignoreFiles) { - const gitignoreDir = dirname(gitignoreFile); // e.g. 'src' or '.' - const content = await readFile(join(cwd, gitignoreFile), 'utf-8'); - filters.push({ dir: gitignoreDir === '.' ? '' : gitignoreDir, ig: makeIg().add(content) }); - } - - // Walk ancestor directories to pick up parent .gitignore files (e.g. monorepo root). - // Check for a .git boundary FIRST so we stop before processing the git root's own - // .gitignore — that file is handled by `git ls-files` when git is available, and - // avoids accidentally applying rules from an unrelated outer repository. - let parentDir = dirname(cwd); - while (parentDir !== dirname(parentDir)) { - if (existsSync(join(parentDir, '.git'))) { - break; - } - - const parentGitignorePath = join(parentDir, '.gitignore'); - if (existsSync(parentGitignorePath)) { - try { - const content = await readFile(parentGitignorePath, 'utf-8'); - // Paths passed to this filter are relative to cwd. To test them against - // a .gitignore that lives above cwd we need to prepend the relative path - // from the ancestor dir to cwd so the ignore patterns see the right scope. - const ancestorPrefix = relative(parentDir, cwd); - filters.push({ dir: '', ig: makeIg().add(content), ancestorPrefix }); - } catch { - // Ignore read errors - } - } + // One `ignore` instance holds every line, scoped to match paths relative to root. The last matching line wins, so + // the files go in from the lowest rank to the highest. + const cwdPath = posixRelative(root, cwd); + const lines: string[] = []; + for (const excludeFile of getGitExcludeFilePaths(cwd, root)) { + lines.push(...keepLinesForCwd(await readFileOrEmpty(excludeFile), cwdPath)); + } - parentDir = dirname(parentDir); + for (const dir of ancestors) { + const content = await readFileOrEmpty(join(dir, '.gitignore')); + lines.push(...scopeLines(keepLinesForCwd(content, posixRelative(dir, cwd)), posixRelative(root, dir))); } - if (filters.length === 0) { - return () => false; + // Git skips a .gitignore it can't read, with a warning. + for (const gitignoreFile of gitignoreFiles) { + const content = await readFileOrEmpty(join(cwd, gitignoreFile)); + lines.push(...scopeLines(content.split(/\r?\n/), posixRelative(root, join(cwd, dirname(gitignoreFile))))); } - return (filePath) => - filters.some(({ dir, ig, ancestorPrefix }) => { - let relativePath: string | null; - if (!dir) { - relativePath = ancestorPrefix ? `${ancestorPrefix}/${filePath}` : filePath; - } else if (filePath.startsWith(`${dir}/`)) { - relativePath = filePath.slice(dir.length + 1); - } else { - relativePath = null; - } - // An empty path is the directory of this .gitignore, which it can't ignore. `ignore` throws on it. - return !!relativePath && ig.ignores(relativePath); - }); + // Names are compared in NFC, as git on macOS prints them, whatever form they have on disk. + const ig = makeIg().add(lines.map((line) => line.normalize('NFC'))); + return (path) => ig.ignores((cwdPath ? `${cwdPath}/${path}` : path).normalize('NFC')); }; /** - * Returns a function that says whether .gitignore rules exclude a path relative to `cwd`. A directory path needs a - * trailing slash. Asks `git ls-files`, which handles ancestor and nested .gitignore files, .git/info/exclude, and the - * global gitignore config. When git fails (not installed, not a repository), reads the .gitignore files itself. + * Returns a function that says whether ignore rules exclude a path relative to `cwd`. A directory path needs a trailing + * slash. Asks `git ls-files`, which applies ancestor and nested .gitignore files, .git/info/exclude, and the global + * excludes file. When git fails, or the rules ignore `cwd` itself, reads the rules itself. * `getActorLocalFilePaths` and the Actor discovery of `apify push` both use it. Discovery asks from the root it - * searches, the upload from the Actor directory. + * searches, the upload from the Actor directory. Discovery names the directories it asks about in `dirs`, with a + * trailing slash: git lists a directory that holds tracked files file by file, even when the rules ignore it, so the + * rules decide about those directories directly. */ -export const getGitignoreMatcher = async (cwd: string): Promise<(path: string) => boolean> => { - try { - const gitIgnored = execSync('git ls-files --others --ignored --exclude-standard --directory', { - cwd, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'ignore'], - }) - .split('\n') - .filter(Boolean); - - if (gitIgnored.length === 0) { - return () => false; - } - - const ig = makeIg().add(gitIgnored); - return (path) => ig.ignores(path); - } catch { - return getGitignoreFallbackMatcher(cwd); - } +export const getGitignoreMatcher = async (cwd: string, dirs: string[] = []): Promise<(path: string) => boolean> => { + // Git can't re-include files below an ignored directory, so it can't say which files the rules inside an ignored cwd + // exclude. Asked from inside, it lists only `./`, or each untracked file when cwd holds tracked ones. + const cwdIgnored = gitOutput('check-ignore -q --no-index ./', cwd) !== null; + // -z prints names as they are. Without it, git quotes names with special characters. + const listed = cwdIgnored + ? null + : gitOutput('ls-files -z --others --ignored --exclude-standard --directory', cwd)?.split('\0').filter(Boolean); + + if (!listed || listed.includes('./')) { + return getGitignoreRulesMatcher(cwd); + } + + // The rules decide about the directories the caller names. Exit code 1 means they ignore none of them. + const ignoredDirs = dirs.length + ? (gitOutput('check-ignore --no-index -z --stdin', cwd, dirs.join('\0'))?.split('\0') ?? []) + : []; + + // Each entry is an exact path, not a pattern, and covers everything below it. A directory entry ends with `/`, but a + // symlink to a directory doesn't. Names are compared in NFC, as git on macOS prints them. + const ignored = new Set([...listed, ...ignoredDirs].map((entry) => entry.normalize('NFC'))); + const prefixes = listed.map((entry) => (entry.endsWith('/') ? entry : `${entry}/`).normalize('NFC')); + return (path) => { + const name = path.normalize('NFC'); + return ignored.has(name) || prefixes.some((prefix) => name.startsWith(prefix)); + }; }; interface ActorIgnoreResult { diff --git a/test/__setup__/hooks/useIsolatedGit.ts b/test/__setup__/hooks/useIsolatedGit.ts new file mode 100644 index 000000000..0d44dd91d --- /dev/null +++ b/test/__setup__/hooks/useIsolatedGit.ts @@ -0,0 +1,43 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; + +import { useTempPath } from './useTempPath.js'; + +/** + * Gives git an empty home, XDG config directory and global config for the tests of a file, so the developer's own + * settings and global excludes file don't apply. `name` names the temp directory that holds them, so it must be unique + * per test file. + */ +export function useIsolatedGit(name: string) { + const gitHome = useTempPath(name, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + /** Points git, and `os.homedir()`, at another home or XDG config directory. Left out, each is the empty one. */ + const stubGitHome = (home = gitHome.joinPath('home'), xdgConfigHome = gitHome.joinPath('xdg')) => { + // os.homedir() reads USERPROFILE on Windows and HOME elsewhere. Git reads HOME everywhere. + vitest.stubEnv('HOME', home); + vitest.stubEnv('USERPROFILE', home); + vitest.stubEnv('XDG_CONFIG_HOME', xdgConfigHome); + vitest.stubEnv('GIT_CONFIG_GLOBAL', gitHome.joinPath('gitconfig')); + vitest.stubEnv('GIT_CONFIG_NOSYSTEM', '1'); + }; + + beforeAll(async () => { + await gitHome.beforeAllCalls(); + + mkdirSync(gitHome.joinPath('home'), { recursive: true }); + mkdirSync(gitHome.joinPath('xdg'), { recursive: true }); + writeFileSync(gitHome.joinPath('gitconfig'), ''); + stubGitHome(); + }); + + afterAll(async () => { + vitest.unstubAllEnvs(); + await gitHome.afterAllCalls(); + }); + + return { stubGitHome }; +} diff --git a/test/local/commands/push-actor-paths.test.ts b/test/local/commands/push-actor-paths.test.ts index 962dafba3..bf0f43ea5 100644 --- a/test/local/commands/push-actor-paths.test.ts +++ b/test/local/commands/push-actor-paths.test.ts @@ -7,6 +7,7 @@ import { testRunCommand } from '../../../src/lib/command-framework/apify-command import { CommandExitCodes } from '../../../src/lib/consts.js'; import { cwdCache } from '../../../src/lib/hooks/useActorConfig.js'; import { useConsoleSpy } from '../../__setup__/hooks/useConsoleSpy.js'; +import { useIsolatedGit } from '../../__setup__/hooks/useIsolatedGit.js'; import { useTempPath } from '../../__setup__/hooks/useTempPath.js'; const actName = 'push-actor-paths-actor'; @@ -55,6 +56,10 @@ const { beforeAllCalls, afterAllCalls, joinPath, tmpPath } = useTempPath(actName cwdParent: false, }); +// The Actor folder sits in test/tmp, which the apify-cli .gitignore ignores, so the push reads the ignore rules itself, +// the global excludes file included. +useIsolatedGit('push-actor-paths-git-home'); + const { logMessages } = useConsoleSpy(); const { ActorsPushCommand } = await import('../../../src/commands/actors/push.js'); diff --git a/test/local/commands/push-monorepo.test.ts b/test/local/commands/push-monorepo.test.ts index d2148be35..7aef37d4d 100644 --- a/test/local/commands/push-monorepo.test.ts +++ b/test/local/commands/push-monorepo.test.ts @@ -9,6 +9,7 @@ import { CommandExitCodes } from '../../../src/lib/consts.js'; import { MONOREPO_PLATFORM_SUPPORT_ISSUE_URL } from '../../../src/lib/docker-context.js'; import { cwdCache } from '../../../src/lib/hooks/useActorConfig.js'; import { useConsoleSpy } from '../../__setup__/hooks/useConsoleSpy.js'; +import { useIsolatedGit } from '../../__setup__/hooks/useIsolatedGit.js'; import { useTempPath } from '../../__setup__/hooks/useTempPath.js'; const actName = 'push-monorepo-actor'; @@ -61,6 +62,9 @@ const { beforeAllCalls, afterAllCalls, joinPath, tmpPath, forceNewCwd } = useTem cwdParent: false, }); +// Git reads the global excludes file, so the developer's own one could leave out the files these tests push. +useIsolatedGit('push-monorepo-git-home'); + const { logMessages } = useConsoleSpy(); const { ActorsPushCommand } = await import('../../../src/commands/actors/push.js'); diff --git a/test/local/commands/push-select.test.ts b/test/local/commands/push-select.test.ts index e4e65be8c..ab604b3fd 100644 --- a/test/local/commands/push-select.test.ts +++ b/test/local/commands/push-select.test.ts @@ -6,6 +6,7 @@ import { testRunCommand } from '../../../src/lib/command-framework/apify-command import { CommandExitCodes, LOCAL_CONFIG_PATH } from '../../../src/lib/consts.js'; import { cwdCache } from '../../../src/lib/hooks/useActorConfig.js'; import { useConsoleSpy } from '../../__setup__/hooks/useConsoleSpy.js'; +import { useIsolatedGit } from '../../__setup__/hooks/useIsolatedGit.js'; import { useTempPath } from '../../__setup__/hooks/useTempPath.js'; // Whether the terminal is interactive is decided by `useStdin` and `ci-info`; both are under test control. @@ -142,6 +143,10 @@ const { beforeAllCalls, afterAllCalls, joinPath, forceNewCwd } = useTempPath('pu cwdParent: false, }); +// The Actors sit in test/tmp, which the apify-cli .gitignore ignores, so the push reads the ignore rules itself, the +// global excludes file included. +useIsolatedGit('push-select-git-home'); + const { logMessages } = useConsoleSpy(); const { ActorsPushCommand } = await import('../../../src/commands/actors/push.js'); diff --git a/test/local/lib/find-actor-dirs.test.ts b/test/local/lib/find-actor-dirs.test.ts index 961801fd1..b923e0b36 100644 --- a/test/local/lib/find-actor-dirs.test.ts +++ b/test/local/lib/find-actor-dirs.test.ts @@ -4,18 +4,26 @@ import { dirname, join } from 'node:path'; import process from 'node:process'; import { findActorDirs, toPosixRelative } from '../../../src/lib/find-actor-dirs.js'; -import { getActorLocalFilePaths } from '../../../src/lib/utils.js'; +import { getActorLocalFilePaths, getGitignoreMatcher } from '../../../src/lib/utils.js'; +import { useIsolatedGit } from '../../__setup__/hooks/useIsolatedGit.js'; import { useTempPath } from '../../__setup__/hooks/useTempPath.js'; // vi.mock is hoisted before imports, so the gitignore matcher gets the mocked version. With `gitAvailable` off, every -// git call fails as it does without git, which sends the matcher to its own .gitignore parsing. -const gitState = vi.hoisted(() => ({ gitAvailable: true })); +// git call fails as it does without git, which sends the matcher to its own .gitignore parsing. `failing` fails only +// the commands it returns true for. `commands` records every command. +const gitState = vi.hoisted(() => ({ + gitAvailable: true, + failing: undefined as ((command: string) => boolean) | undefined, + commands: [] as string[], +})); vi.mock('node:child_process', async (importOriginal) => { const original = await importOriginal(); return { ...original, execSync: ((...args: Parameters) => { - if (!gitState.gitAvailable) { + const command = String(args[0]); + gitState.commands.push(command); + if (!gitState.gitAvailable || gitState.failing?.(command)) { throw new Error('not a git repository'); } return original.execSync(...args); @@ -23,6 +31,8 @@ vi.mock('node:child_process', async (importOriginal) => { }; }); +useIsolatedGit('find-actor-dirs-git-home'); + const { beforeAllCalls, afterAllCalls, joinPath, tmpPath } = useTempPath('find-actor-dirs', { create: true, remove: true, @@ -41,7 +51,11 @@ async function writeActor(dir: string, name: string | undefined = dir.split('/') beforeEach(async () => { gitState.gitAvailable = true; + gitState.failing = undefined; await beforeAllCalls(); + // Its own repository, so the .gitignore and .git/info/exclude of the apify-cli checkout don't apply. + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + gitState.commands = []; }); afterEach(async () => { @@ -136,11 +150,14 @@ describe('findActorDirs', () => { describe('Actors in directories .gitignore ignores', () => { // The root .gitignore ignores a parent directory, and a nested .gitignore ignores an Actor directory itself. // .git/info/exclude ignores actors/delta, which only git reads. An Actor's own .gitignore, as every template - // ships, and an ignored file inside an Actor leave the Actor alone. + // ships, and an ignored file inside an Actor leave the Actor alone. The root .gitignore also ignores legacy, + // which holds a tracked file, so git lists its other files one by one. async function writeRepository() { - execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); await write('.git/info/exclude', 'actors/delta/\n'); - await write('.gitignore', 'archive/\n*.log\n'); + await write('.gitignore', 'archive/\nlegacy/\n*.log\n'); + await writeActor('legacy'); + await write('legacy/main.js'); + execSync('git add -f legacy/main.js', { cwd: tmpPath, stdio: 'ignore' }); await write('actors/.gitignore', 'gamma\n'); await writeActor('actors/beta'); await write('actors/beta/.gitignore', 'node_modules\nstorage\n'); @@ -155,6 +172,7 @@ describe('findActorDirs', () => { { dir: 'actors/delta', name: 'delta', ignored: deltaIgnored }, { dir: 'actors/gamma', name: 'gamma', ignored: true }, { dir: 'archive/old/alpha', name: 'alpha', ignored: true }, + { dir: 'legacy', name: 'legacy', ignored: true }, ]; it('asks git and marks them ignored, as the upload leaves them out', async () => { @@ -167,6 +185,7 @@ describe('findActorDirs', () => { expect(uploaded).not.toContain('actors/delta/.actor/actor.json'); expect(uploaded).not.toContain('actors/gamma/.actor/actor.json'); expect(uploaded).not.toContain('archive/old/alpha/.actor/actor.json'); + expect(uploaded).not.toContain('legacy/.actor/actor.json'); }); it('reads the .gitignore files itself when git is unavailable', async () => { @@ -175,5 +194,80 @@ describe('findActorDirs', () => { await expect(findActorDirs(tmpPath)).resolves.toEqual(expected({ deltaIgnored: false })); }); + + // In the next two, the Actor directory holds a tracked file, so git lists its other files one by one and is + // asked about the directory itself. + it('asks git about an ignored Actor with a non-ASCII name that holds a tracked file', async () => { + // git's default, set here in case a config turns it off. Without -z, git prints such names quoted and + // escaped. + execSync('git config core.quotePath true', { cwd: tmpPath, stdio: 'ignore' }); + await write('.gitignore', 'old/\n'); + await writeActor('old/příliš'); + await write('old/příliš/main.js'); + execSync('git add -f old', { cwd: tmpPath, stdio: 'ignore' }); + + await expect(findActorDirs(tmpPath)).resolves.toEqual([{ dir: 'old/příliš', name: 'příliš', ignored: true }]); + }); + + it('asks git about an ignored Actor whose name is stored decomposed', async () => { + // Decomposed, as macOS archives and older macOS volumes store names. Git prints the name back as it was + // asked. + const name = 'čaj'.normalize('NFD'); + await write('.gitignore', 'old/\n'); + await writeActor(`old/${name}`); + await write(`old/${name}/main.js`); + execSync('git add -f old', { cwd: tmpPath, stdio: 'ignore' }); + + const found = await findActorDirs(tmpPath); + + expect(found.map((actor) => ({ ...actor, dir: actor.dir.normalize('NFC') }))).toEqual([ + { dir: 'old/čaj', name, ignored: true }, + ]); + }); + + it('judges from a root that .gitignore ignores, even when it holds a tracked file', async () => { + // Discovery from group/ drops the line that ignores group/ itself, as the upload does, and keeps the rest. + await write('.gitignore', 'group/\n'); + await write('group/.gitignore', 'old/\n'); + await write('group/README.md'); + execSync('git add -f group/README.md', { cwd: tmpPath, stdio: 'ignore' }); + await writeActor('group/actors/alpha'); + await writeActor('group/old/beta'); + + await expect(findActorDirs(joinPath('group'))).resolves.toEqual([ + { dir: 'actors/alpha', name: 'alpha', ignored: false }, + { dir: 'old/beta', name: 'beta', ignored: true }, + ]); + }); + }); +}); + +describe('getGitignoreMatcher, which discovery shares with the upload', () => { + it('asks git about directories only when the caller names some', async () => { + await writeActor('actors/alpha'); + const stdinCommands = () => gitState.commands.filter((command) => command.includes('--stdin')); + + await getActorLocalFilePaths(tmpPath); + expect(stdinCommands()).toEqual([]); + + await findActorDirs(tmpPath); + expect(stdinCommands()).toHaveLength(1); + }); + + it('reads the rules itself when git lists ./ from a directory .gitignore ignores', async () => { + // The check whether the rules ignore the directory fails, so git is asked for its listing, which is only `./`. + await write('.gitignore', 'ghost/\n*.log\n'); + await write('ghost/.gitignore', 'secret.env\n'); + await write('ghost/main.js'); + await write('ghost/secret.env'); + await write('ghost/x.log'); + gitState.failing = (command) => command.includes('check-ignore') && command.endsWith(' ./'); + + const isGitignored = await getGitignoreMatcher(joinPath('ghost')); + + expect(gitState.commands.some((command) => command.includes('ls-files'))).toBe(true); + expect(isGitignored('main.js')).toBe(false); + expect(isGitignored('secret.env')).toBe(true); + expect(isGitignored('x.log')).toBe(true); }); }); diff --git a/test/local/lib/utils-actorignore-no-git.test.ts b/test/local/lib/utils-actorignore-no-git.test.ts index 240bcf278..970131063 100644 --- a/test/local/lib/utils-actorignore-no-git.test.ts +++ b/test/local/lib/utils-actorignore-no-git.test.ts @@ -1,4 +1,4 @@ -import { writeFileSync } from 'node:fs'; +import { mkdirSync, writeFileSync } from 'node:fs'; import { ensureFolderExistsSync } from '../../../src/lib/files.js'; import { getActorLocalFilePaths } from '../../../src/lib/utils.js'; @@ -32,6 +32,9 @@ describe('Utils - .actorignore with .gitignore fallback (no git)', () => { beforeAll(async () => { await beforeAllCalls(); + // The fake .git stops the walk here, so the apify-cli .gitignore doesn't apply. + mkdirSync(joinPath('.git')); + FOLDERS.forEach((folder) => { ensureFolderExistsSync(tmpPath, folder); }); @@ -73,6 +76,9 @@ describe('Utils - .actorignore negation overrides gitignore (no git)', () => { beforeAll(async () => { await beforeAllCalls(); + // The fake .git stops the walk here, so the apify-cli .gitignore doesn't apply. + mkdirSync(joinPath('.git')); + NEGATE_FOLDERS.forEach((folder) => { ensureFolderExistsSync(tmpPath, folder); }); diff --git a/test/local/lib/utils-actorignore.test.ts b/test/local/lib/utils-actorignore.test.ts index 7d1311bef..d5555b6b9 100644 --- a/test/local/lib/utils-actorignore.test.ts +++ b/test/local/lib/utils-actorignore.test.ts @@ -1,10 +1,14 @@ -import { execSync } from 'node:child_process'; -import { writeFileSync } from 'node:fs'; +import { execSync, spawnSync } from 'node:child_process'; +import { mkdirSync, symlinkSync, writeFileSync } from 'node:fs'; +import process from 'node:process'; import { ensureFolderExistsSync } from '../../../src/lib/files.js'; -import { getActorLocalFilePaths } from '../../../src/lib/utils.js'; +import { getActorLocalFilePaths, getGitignoreMatcher } from '../../../src/lib/utils.js'; +import { useIsolatedGit } from '../../__setup__/hooks/useIsolatedGit.js'; import { useTempPath } from '../../__setup__/hooks/useTempPath.js'; +const { stubGitHome } = useIsolatedGit('actorignore-git-home'); + const TEST_DIR = 'actorignore-test-dir'; const FOLDERS = ['src', 'docs', 'assets']; const FILES = ['main.js', 'src/index.js']; @@ -368,3 +372,811 @@ describe('Utils - no .actorignore present (git)', () => { expect(paths).toContain('docs/README.md'); }); }); + +const NON_ASCII_TEST_DIR = 'gitignore-non-ascii-test-dir'; + +describe('Utils - gitignored files with non-ASCII names (git)', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(NON_ASCII_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + // git's default, set here in case a global config turns it off. It prints such names quoted and escaped. + execSync('git config core.quotePath true', { cwd: tmpPath, stdio: 'ignore' }); + // git's default on macOS, where it prints names in NFC whatever their form on disk. Other systems ignore it. + execSync('git config core.precomposeUnicode true', { cwd: tmpPath, stdio: 'ignore' }); + + ensureFolderExistsSync(tmpPath, 'sub'); + + writeFileSync(joinPath('main.js'), 'content', { flag: 'w' }); + writeFileSync(joinPath('příliš.log'), 'content', { flag: 'w' }); + writeFileSync(joinPath('sub/žluťoučký.log'), 'content', { flag: 'w' }); + // Decomposed, as macOS archives and older macOS volumes store names. + writeFileSync(joinPath('čaj.log'.normalize('NFD')), 'content', { flag: 'w' }); + + writeFileSync(joinPath('.gitignore'), '*.log\n', { flag: 'w' }); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should exclude them', async () => { + const paths = (await getActorLocalFilePaths(tmpPath)).map((path) => path.normalize('NFC')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('příliš.log'); + expect(paths).not.toContain('sub/žluťoučký.log'); + expect(paths).not.toContain('čaj.log'); + }); +}); + +const IGNORED_SYMLINK_TEST_DIR = 'gitignore-ignored-symlink-test-dir'; + +// Creating a symlink on Windows needs extra rights. +describe.skipIf(process.platform === 'win32')('Utils - gitignored symlink to a directory (git)', () => { + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(IGNORED_SYMLINK_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('repo'), { recursive: true }); + mkdirSync(joinPath('outside'), { recursive: true }); + execSync('git init', { cwd: joinPath('repo'), stdio: 'ignore' }); + + writeFileSync(joinPath('outside', 'o1.txt'), 'content', { flag: 'w' }); + writeFileSync(joinPath('repo', 'main.js'), 'content', { flag: 'w' }); + symlinkSync(joinPath('outside'), joinPath('repo', 'data'), 'dir'); + + // git lists the ignored symlink as `data`, without the trailing slash of a directory. + writeFileSync(joinPath('repo', '.gitignore'), 'data\n', { flag: 'w' }); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should exclude the files below it', async () => { + const paths = await getActorLocalFilePaths(joinPath('repo')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('data/o1.txt'); + }); +}); + +const PATTERN_NAMES_TEST_DIR = 'gitignore-pattern-names-test-dir'; + +describe('Utils - gitignored files whose names look like patterns (git)', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(PATTERN_NAMES_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + + writeFileSync(joinPath('a.js'), 'content', { flag: 'w' }); + writeFileSync(joinPath('#hash.txt'), 'content', { flag: 'w' }); + writeFileSync(joinPath('[ab].js'), 'content', { flag: 'w' }); + + // Escaped, so both lines match the literal names only. + writeFileSync(joinPath('.gitignore'), '\\#hash.txt\n\\[ab\\].js\n', { flag: 'w' }); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should exclude exactly the files git lists', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).not.toContain('#hash.txt'); + expect(paths).not.toContain('[ab].js'); + expect(paths).toContain('a.js'); + }); +}); + +const GIT_FORCE_INCLUDE_TEST_DIR = 'actorignore-git-force-include-test-dir'; + +describe('Utils - .actorignore negation with directories and files git lists (git)', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(GIT_FORCE_INCLUDE_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + + mkdirSync(joinPath('build', 'nested'), { recursive: true }); + + writeFileSync(joinPath('main.js'), 'content', { flag: 'w' }); + writeFileSync(joinPath('build/out.js'), 'content', { flag: 'w' }); + writeFileSync(joinPath('build/nested/chunk.js'), 'content', { flag: 'w' }); + writeFileSync(joinPath('keep.log'), 'content', { flag: 'w' }); + writeFileSync(joinPath('drop.log'), 'content', { flag: 'w' }); + + // git lists build/ as a directory and the two logs as files. + writeFileSync(joinPath('.gitignore'), 'build/\n*.log\n', { flag: 'w' }); + writeFileSync(joinPath('.actorignore'), '!build/\n!keep.log\n', { flag: 'w' }); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should include the force-included files and exclude the rest', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('main.js'); + expect(paths).toContain('build/out.js'); + expect(paths).toContain('build/nested/chunk.js'); + expect(paths).toContain('keep.log'); + expect(paths).not.toContain('drop.log'); + }); +}); + +const IGNORED_DIR_GIT_TEST_DIR = 'gitignore-ignored-dir-git-test-dir'; + +describe('Utils - pushing a directory .gitignore ignores (git)', () => { + // The root .gitignore ignores ghost/, so git lists only `./` from inside it. The push is explicit, so every other + // rule still applies: the root .gitignore, ghost/.gitignore, .git/info/exclude, and the excludes file git config + // names. ghost/.actorignore force-includes a directory ghost/.gitignore ignores. The `!` lines of a deeper file + // re-include what a shallower one ignores. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(IGNORED_DIR_GIT_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + execSync('git config core.excludesFile global-excludes', { cwd: tmpPath, stdio: 'ignore' }); + + mkdirSync(joinPath('ghost', 'src'), { recursive: true }); + mkdirSync(joinPath('ghost', 'dist'), { recursive: true }); + + writeFileSync(joinPath('.gitignore'), 'ghost/\n*.log\n', { flag: 'w' }); + writeFileSync(joinPath('.git', 'info', 'exclude'), '*.pem\n', { flag: 'w' }); + writeFileSync(joinPath('global-excludes'), '*.bak\n', { flag: 'w' }); + writeFileSync(joinPath('ghost', '.gitignore'), 'secret.env\ndist/\n!public.pem\n', { flag: 'w' }); + writeFileSync(joinPath('ghost', 'src', '.gitignore'), '!keep.log\n', { flag: 'w' }); + writeFileSync(joinPath('ghost', '.actorignore'), '!dist/\n', { flag: 'w' }); + + [ + 'main.js', + 'src/index.js', + 'secret.env', + 'x.log', + 'src/y.log', + 'src/keep.log', + 'key.pem', + 'public.pem', + 'notes.bak', + 'dist/bundle.js', + ].forEach((file) => writeFileSync(joinPath('ghost', file), 'content', { flag: 'w' })); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should exclude the files its own .gitignore lists', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost')); + + expect(paths).toContain('main.js'); + expect(paths).toContain('src/index.js'); + expect(paths).not.toContain('secret.env'); + }); + + it('should exclude the files the rules above it list', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost')); + + expect(paths).not.toContain('x.log'); + expect(paths).not.toContain('src/y.log'); + expect(paths).not.toContain('key.pem'); + expect(paths).not.toContain('notes.bak'); + }); + + it('should apply the rules above it in a subdirectory too', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost', 'src')); + + expect(paths).toContain('index.js'); + expect(paths).not.toContain('y.log'); + }); + + it('should let a deeper file re-include what a shallower one ignores, as git does', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost')); + + expect(paths).toContain('src/keep.log'); + expect(paths).toContain('public.pem'); + expect(await getActorLocalFilePaths(joinPath('ghost', 'src'))).toContain('keep.log'); + }); + + it('should include the files .actorignore force-includes', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost')); + + expect(paths).toContain('dist/bundle.js'); + }); +}); + +const IGNORED_TRACKED_DIR_TEST_DIR = 'gitignore-ignored-tracked-dir-test-dir'; + +describe('Utils - pushing a directory .gitignore ignores that holds a tracked file (git)', () => { + // The root .gitignore ignores legacy/, which holds a file committed before. git lists the untracked files in it one by + // one, not `./`, but the push applies the rules as it does for any ignored directory. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(IGNORED_TRACKED_DIR_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + + ensureFolderExistsSync(tmpPath, 'legacy'); + + writeFileSync(joinPath('.gitignore'), 'legacy/\n*.log\n', { flag: 'w' }); + writeFileSync(joinPath('legacy', '.gitignore'), 'secret.env\n', { flag: 'w' }); + ['tracked.js', 'main.js', 'secret.env', 'x.log'].forEach((file) => + writeFileSync(joinPath('legacy', file), 'content', { flag: 'w' }), + ); + + execSync('git add -f legacy/tracked.js', { cwd: tmpPath, stdio: 'ignore' }); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply every rule except the one that ignores the pushed directory', async () => { + const paths = await getActorLocalFilePaths(joinPath('legacy')); + + expect(paths).toContain('tracked.js'); + expect(paths).toContain('main.js'); + expect(paths).toContain('.gitignore'); + expect(paths).not.toContain('secret.env'); + expect(paths).not.toContain('x.log'); + }); +}); + +const IGNORED_DIR_SYMLINK_TEST_DIR = 'gitignore-ignored-dir-symlink-test-dir'; + +// Creating a symlink on Windows needs extra rights. +describe.skipIf(process.platform === 'win32')( + 'Utils - pushing a directory .gitignore ignores through a symlink (git)', + () => { + // The root .gitignore ignores actors/, and linked leads to actors/a. Git applies the rules by the real path, so + // a push through the link applies the same rules as a push of actors/a. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(IGNORED_DIR_SYMLINK_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + + mkdirSync(joinPath('actors', 'a', 'local'), { recursive: true }); + + writeFileSync(joinPath('.gitignore'), 'actors/\n/actors/a/local/\n', { flag: 'w' }); + writeFileSync(joinPath('.git', 'info', 'exclude'), '*.pem\n', { flag: 'w' }); + ['main.js', 'local/x.txt', 'key.pem'].forEach((file) => + writeFileSync(joinPath('actors', 'a', file), 'content', { flag: 'w' }), + ); + symlinkSync(joinPath('actors', 'a'), joinPath('linked'), 'dir'); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply the rules by the real path', async () => { + const paths = await getActorLocalFilePaths(joinPath('linked')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('local/x.txt'); + expect(paths).not.toContain('key.pem'); + }); + }, +); + +const TRACKED_FILE_TEST_DIR = 'gitignore-tracked-file-test-dir'; + +describe('Utils - tracked file the .gitignore matches (git)', () => { + // Git lists only untracked files, so a file committed before the rule stays in the upload, as it stays in the + // repository. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(TRACKED_FILE_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + + ['main.js', 'keep.log', 'other.log'].forEach((file) => writeFileSync(joinPath(file), 'content', { flag: 'w' })); + writeFileSync(joinPath('.gitignore'), '*.log\n', { flag: 'w' }); + + execSync('git add -f keep.log', { cwd: tmpPath, stdio: 'ignore' }); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should keep the tracked file, as git lists only untracked files', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('main.js'); + expect(paths).toContain('keep.log'); + expect(paths).not.toContain('other.log'); + }); +}); + +const MATCHER_TEST_DIR = 'gitignore-matcher-test-dir'; + +describe('Utils - getGitignoreMatcher asked directly (git)', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(MATCHER_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + + ensureFolderExistsSync(tmpPath, 'dist'); + ensureFolderExistsSync(tmpPath, 'src'); + + ['dist/bundle.js', 'src/index.js', 'x.log', 'x.logs'].forEach((file) => + writeFileSync(joinPath(file), 'content', { flag: 'w' }), + ); + writeFileSync(joinPath('.gitignore'), 'dist/\n*.log\n', { flag: 'w' }); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should say an ignored directory asked with a trailing slash is ignored', async () => { + const isGitignored = await getGitignoreMatcher(tmpPath); + + expect(isGitignored('dist/')).toBe(true); + expect(isGitignored('src/')).toBe(false); + }); + + it('should not say a path that only starts with the name of an ignored file is ignored', async () => { + const isGitignored = await getGitignoreMatcher(tmpPath); + + expect(isGitignored('x.log')).toBe(true); + expect(isGitignored('x.logs')).toBe(false); + }); + + it('should drop the empty entry after the last NUL of the git listing', async () => { + const isGitignored = await getGitignoreMatcher(tmpPath); + + expect(isGitignored('x.log')).toBe(true); + // -z ends the listing with a NUL, so the split leaves an empty entry. '' is the only path it would match. + expect(isGitignored('')).toBe(false); + }); +}); + +const NFD_DIR_TEST_DIR = 'gitignore-nfd-dir-test-dir'; + +describe('Utils - gitignored directory with a decomposed non-ASCII name (git)', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(NFD_DIR_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + // Git then prints names as stored, as it does on Linux and Windows. + execSync('git config core.precomposeUnicode false', { cwd: tmpPath, stdio: 'ignore' }); + + // Decomposed, as macOS archives and older macOS volumes store names. + ensureFolderExistsSync(tmpPath, 'čaj'.normalize('NFD')); + + writeFileSync(joinPath('main.js'), 'content', { flag: 'w' }); + writeFileSync(joinPath('čaj'.normalize('NFD'), 'a.txt'), 'content', { flag: 'w' }); + + // ASCII, so it matches either form. + writeFileSync(joinPath('.gitignore'), '*aj/\n', { flag: 'w' }); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should exclude the files below it', async () => { + const paths = (await getActorLocalFilePaths(tmpPath)).map((path) => path.normalize('NFC')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('čaj/a.txt'); + }); +}); + +const EXCLUDE_RANK_TEST_DIR = 'gitignore-exclude-rank-test-dir'; + +describe('Utils - pushing a directory .gitignore ignores ranks the exclude files as git does (git)', () => { + // .git/info/exclude ranks above the global excludes file, and every .gitignore above both. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(EXCLUDE_RANK_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + execSync('git config core.excludesFile global-excludes', { cwd: tmpPath, stdio: 'ignore' }); + + ensureFolderExistsSync(tmpPath, 'ghost'); + + writeFileSync(joinPath('global-excludes'), '*.bak\n', { flag: 'w' }); + writeFileSync(joinPath('.git', 'info', 'exclude'), '!keep.bak\n*.pem\n', { flag: 'w' }); + writeFileSync(joinPath('.gitignore'), 'ghost/\n!public.pem\n', { flag: 'w' }); + ['main.js', 'keep.bak', 'other.bak', 'public.pem', 'key.pem'].forEach((file) => + writeFileSync(joinPath('ghost', file), 'content', { flag: 'w' }), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should let .git/info/exclude re-include what the global excludes file ignores', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost')); + + expect(paths).toContain('main.js'); + expect(paths).toContain('keep.bak'); + expect(paths).not.toContain('other.bak'); + }); + + it('should let a .gitignore re-include what .git/info/exclude ignores', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost')); + + expect(paths).toContain('public.pem'); + expect(paths).not.toContain('key.pem'); + }); +}); + +const IGNORING_LINES_TEST_DIR = 'gitignore-ignoring-lines-test-dir'; + +describe('Utils - pushing a directory a line above it ignores (git)', () => { + // Whatever the form of the line that ignores the pushed directory, only that line is dropped. The other lines of its + // file still apply, its ! lines among them. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(IGNORING_LINES_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + + ['ghost', 'actors/a', 'actors/delta'].forEach((folder) => mkdirSync(joinPath(folder), { recursive: true })); + + writeFileSync(joinPath('.gitignore'), 'ghost*\na/\n*.log\n!keep.log\n', { flag: 'w' }); + writeFileSync(joinPath('.git', 'info', 'exclude'), 'actors/delta/\n*.pem\n', { flag: 'w' }); + [ + 'ghost/main.js', + 'ghost/x.log', + 'ghost/keep.log', + 'actors/a/main.js', + 'actors/a/x.log', + 'actors/delta/main.js', + 'actors/delta/key.pem', + ].forEach((file) => writeFileSync(joinPath(file), 'content', { flag: 'w' })); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should drop a line without a trailing slash', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('x.log'); + }); + + it('should drop a line without a slash that matches at any depth', async () => { + const paths = await getActorLocalFilePaths(joinPath('actors', 'a')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('x.log'); + }); + + it('should keep the ! lines of the file', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost')); + + expect(paths).toContain('keep.log'); + }); + + it('should drop the line of .git/info/exclude', async () => { + const paths = await getActorLocalFilePaths(joinPath('actors', 'delta')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('key.pem'); + }); +}); + +const NOT_A_REPOSITORY_TEST_DIR = 'gitignore-not-a-repository-test-dir'; + +describe('Utils - pushing an ignored directory with a .git that is not a repository (git)', () => { + // inner/.git is an empty directory, so git answers from the outer repository, whose .gitignore ignores inner/. The + // rules are read from inner/, the first directory with a .git, and the outer .git/info/exclude applies there. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(NOT_A_REPOSITORY_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + execSync('git init', { cwd: tmpPath, stdio: 'ignore' }); + + mkdirSync(joinPath('inner', '.git'), { recursive: true }); + + writeFileSync(joinPath('.gitignore'), 'inner/\n', { flag: 'w' }); + writeFileSync(joinPath('.git', 'info', 'exclude'), '*.pem\n', { flag: 'w' }); + ['main.js', 'key.pem'].forEach((file) => writeFileSync(joinPath('inner', file), 'content', { flag: 'w' })); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply .git/info/exclude', async () => { + const paths = await getActorLocalFilePaths(joinPath('inner')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('key.pem'); + }); +}); + +const LINKED_WORKTREE_TEST_DIR = 'gitignore-linked-worktree-test-dir'; + +describe('Utils - pushing a directory .gitignore ignores in a linked worktree (git)', () => { + // In a linked worktree .git is a file, and .git/info/exclude is the main repository's. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(LINKED_WORKTREE_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('main')); + execSync('git init', { cwd: joinPath('main'), stdio: 'ignore' }); + execSync('git -c user.name=Test -c user.email=test@example.com commit --quiet --allow-empty -m init', { + cwd: joinPath('main'), + stdio: 'ignore', + }); + execSync(`git worktree add --detach "${joinPath('linked')}"`, { cwd: joinPath('main'), stdio: 'ignore' }); + + mkdirSync(joinPath('linked', 'ghost')); + + writeFileSync(joinPath('main', '.git', 'info', 'exclude'), '*.pem\n', { flag: 'w' }); + writeFileSync(joinPath('linked', '.gitignore'), 'ghost/\n', { flag: 'w' }); + ['main.js', 'key.pem'].forEach((file) => + writeFileSync(joinPath('linked', 'ghost', file), 'content', { flag: 'w' }), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should exclude the files the main repository .git/info/exclude lists', async () => { + const paths = await getActorLocalFilePaths(joinPath('linked', 'ghost')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('key.pem'); + }); +}); + +const GLOBAL_EXCLUDES_TEST_DIR = 'gitignore-global-excludes-test-dir'; + +describe('Utils - the global excludes file when pushing a directory .gitignore ignores (git)', () => { + // The root .gitignore ignores ghost/, so the push finds the global excludes file itself, as git does. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(GLOBAL_EXCLUDES_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('repo', 'ghost'), { recursive: true }); + execSync('git init', { cwd: joinPath('repo'), stdio: 'ignore' }); + + writeFileSync(joinPath('repo', '.gitignore'), 'ghost/\n', { flag: 'w' }); + ['main.js', 'notes.bak'].forEach((file) => + writeFileSync(joinPath('repo', 'ghost', file), 'content', { flag: 'w' }), + ); + }); + + beforeEach(() => { + try { + execSync('git config --unset-all core.excludesFile', { cwd: joinPath('repo'), stdio: 'ignore' }); + } catch { + // It wasn't set. + } + }); + + afterEach(() => { + stubGitHome(); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply no global excludes when the file does not exist', async () => { + // The empty home holds no git/ignore. + const paths = await getActorLocalFilePaths(joinPath('repo', 'ghost')); + + expect(paths).toContain('main.js'); + expect(paths).toContain('notes.bak'); + }); + + it('should read core.excludesFile when it is an absolute path', async () => { + writeFileSync(joinPath('elsewhere-excludes'), '*.bak\n', { flag: 'w' }); + execSync(`git config core.excludesFile "${joinPath('elsewhere-excludes').replaceAll('\\', '/')}"`, { + cwd: joinPath('repo'), + stdio: 'ignore', + }); + + const paths = await getActorLocalFilePaths(joinPath('repo', 'ghost')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('notes.bak'); + }); + + it('should expand a leading ~ in core.excludesFile to the home directory', async () => { + mkdirSync(joinPath('home'), { recursive: true }); + writeFileSync(joinPath('home', 'my-excludes'), '*.bak\n', { flag: 'w' }); + stubGitHome(joinPath('home')); + execSync('git config core.excludesFile "~/my-excludes"', { cwd: joinPath('repo'), stdio: 'ignore' }); + + const paths = await getActorLocalFilePaths(joinPath('repo', 'ghost')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('notes.bak'); + }); + + it('should read $XDG_CONFIG_HOME/git/ignore when core.excludesFile is unset', async () => { + mkdirSync(joinPath('xdg', 'git'), { recursive: true }); + writeFileSync(joinPath('xdg', 'git', 'ignore'), '*.bak\n', { flag: 'w' }); + stubGitHome(undefined, joinPath('xdg')); + + const paths = await getActorLocalFilePaths(joinPath('repo', 'ghost')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('notes.bak'); + }); + + it('should read ~/.config/git/ignore when XDG_CONFIG_HOME is unset', async () => { + mkdirSync(joinPath('home-unset', '.config', 'git'), { recursive: true }); + writeFileSync(joinPath('home-unset', '.config', 'git', 'ignore'), '*.bak\n', { flag: 'w' }); + stubGitHome(joinPath('home-unset')); + vitest.stubEnv('XDG_CONFIG_HOME', undefined); + + const paths = await getActorLocalFilePaths(joinPath('repo', 'ghost')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('notes.bak'); + }); + + it('should apply no global excludes when core.excludesFile is empty, as git does', async () => { + mkdirSync(joinPath('xdg-off', 'git'), { recursive: true }); + writeFileSync(joinPath('xdg-off', 'git', 'ignore'), '*.bak\n', { flag: 'w' }); + stubGitHome(undefined, joinPath('xdg-off')); + spawnSync('git', ['config', 'core.excludesFile', ''], { cwd: joinPath('repo'), stdio: 'ignore' }); + + const paths = await getActorLocalFilePaths(joinPath('repo', 'ghost')); + + expect(paths).toContain('main.js'); + expect(paths).toContain('notes.bak'); + }); + + it('should read ~/.config/git/ignore when XDG_CONFIG_HOME is empty, as git does', async () => { + mkdirSync(joinPath('home-empty', '.config', 'git'), { recursive: true }); + writeFileSync(joinPath('home-empty', '.config', 'git', 'ignore'), '*.bak\n', { flag: 'w' }); + stubGitHome(joinPath('home-empty'), ''); + + const paths = await getActorLocalFilePaths(joinPath('repo', 'ghost')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('notes.bak'); + }); +}); + +const GIT_ERRORS_TEST_DIR = 'gitignore-git-errors-test-dir'; + +describe('Utils - git errors (git)', () => { + const { tmpPath, beforeAllCalls, afterAllCalls } = useTempPath(GIT_ERRORS_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should not print them when git fails', () => { + // git writes to the terminal directly, so the push runs in a process of its own. GIT_DIR naming a missing + // directory makes every git command fail with "fatal: not a git repository". + const utils = new URL('../../../src/lib/utils.ts', import.meta.url).href; + const script = `const { getActorLocalFilePaths } = await import(${JSON.stringify(utils)}); await getActorLocalFilePaths(process.cwd());`; + const { status, stderr } = spawnSync( + process.execPath, + ['--import', 'tsx', '--input-type=module', '--eval', script], + { + cwd: tmpPath, + env: { ...process.env, GIT_DIR: `${tmpPath}-missing` }, + encoding: 'utf-8', + }, + ); + + expect(status).toBe(0); + expect(stderr).not.toContain('fatal'); + }); +}); diff --git a/test/local/lib/utils-gitignore-fallback.test.ts b/test/local/lib/utils-gitignore-fallback.test.ts index aa12d8aef..00e62d9de 100644 --- a/test/local/lib/utils-gitignore-fallback.test.ts +++ b/test/local/lib/utils-gitignore-fallback.test.ts @@ -1,5 +1,8 @@ -import { mkdirSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, sep } from 'node:path'; +import process from 'node:process'; import { ensureFolderExistsSync } from '../../../src/lib/files.js'; import { getActorLocalFilePaths } from '../../../src/lib/utils.js'; @@ -17,6 +20,12 @@ vi.mock('node:child_process', async (importOriginal) => { }; }); +// Lets a test see which files the push reads. +vi.mock('node:fs/promises', async (importOriginal) => { + const original = await importOriginal(); + return { ...original, readFile: vi.fn(original.readFile) }; +}); + const TEST_DIR = 'gitignore-fallback-test-dir'; const FOLDERS = ['src', 'src/utils']; const FOLDERS_TO_IGNORE = ['dist', 'src/generated']; @@ -36,6 +45,8 @@ describe('Utils - gitignore fallback (no git)', () => { await beforeAllCalls(); // NOTE: No git init here — execSync is mocked to throw, triggering the fallback path. + // The fake .git stops the walk here, so the apify-cli .gitignore doesn't apply. + mkdirSync(joinPath('.git')); FOLDERS.concat(FOLDERS_TO_IGNORE).forEach((folder) => { ensureFolderExistsSync(tmpPath, folder); @@ -74,6 +85,9 @@ describe('Utils - nested .gitignore scoping (no git)', () => { beforeAll(async () => { await beforeAllCalls(); + // The fake .git stops the walk here, so the apify-cli .gitignore doesn't apply. + mkdirSync(joinPath('.git')); + // Create directory structure ensureFolderExistsSync(tmpPath, 'src'); ensureFolderExistsSync(tmpPath, 'src/internal'); @@ -121,8 +135,9 @@ describe('Utils - parent .gitignore applied to subproject (no git)', () => { subprojectPath = join(tmpPath, 'subproject'); // Parent .gitignore — rules that should apply to everything inside subproject/. - // No fake .git is needed: the ancestor-walker already stops at the apify-cli - // repo root (.git lives there) before touching its own .gitignore. + // The fake .git makes tmpPath the repository root, so the walk reads its + // .gitignore and stops there, before the apify-cli .gitignore. + mkdirSync(join(tmpPath, '.git')); writeFileSync(join(tmpPath, '.gitignore'), '*.secret\nbuild/\n', { flag: 'w' }); // Subproject directory structure @@ -160,3 +175,906 @@ describe('Utils - parent .gitignore applied to subproject (no git)', () => { expect(paths).not.toContain('build/output.js'); }); }); + +const ROOT_TEST_DIR = 'gitignore-root-test-dir'; + +describe('Utils - repository root .gitignore applied to a subdirectory (no git)', () => { + // repo/ has a .git, so it's the repository root. The .gitignore above it belongs to an outer repository. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(ROOT_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('repo', '.git'), { recursive: true }); + mkdirSync(joinPath('repo', 'actors', 'a', 'root-ignored'), { recursive: true }); + writeFileSync(joinPath('.gitignore'), 'outer.txt\n'); + writeFileSync(joinPath('repo', '.gitignore'), '*.log\nroot-ignored/\n'); + ['main.js', 'x.log', 'root-ignored/f.txt', 'outer.txt'].forEach((file) => + writeFileSync(joinPath('repo', 'actors', 'a', file), 'content'), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should exclude files matched by the repository root .gitignore, but not by an outer one', async () => { + const paths = await getActorLocalFilePaths(joinPath('repo', 'actors', 'a')); + + expect(paths).toContain('main.js'); + expect(paths).toContain('outer.txt'); + expect(paths).not.toContain('x.log'); + expect(paths).not.toContain('root-ignored/f.txt'); + }); + + it('should not apply the outer .gitignore when the pushed directory is the repository root', async () => { + const paths = await getActorLocalFilePaths(joinPath('repo')); + + expect(paths).toContain('actors/a/outer.txt'); + expect(paths).not.toContain('actors/a/x.log'); + }); +}); + +const IGNORED_DIR_TEST_DIR = 'gitignore-ignored-dir-test-dir'; + +describe('Utils - pushing a directory .gitignore ignores (no git)', () => { + // The root .gitignore ignores ghost/, which is pushed explicitly. Its other rules still apply, and so does + // ghost/.gitignore. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(IGNORED_DIR_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git'), { recursive: true }); + mkdirSync(joinPath('ghost', 'src'), { recursive: true }); + writeFileSync(joinPath('.gitignore'), 'ghost/\n*.log\n'); + writeFileSync(joinPath('ghost', '.gitignore'), 'secret.env\n'); + ['main.js', 'src/index.js', 'secret.env', 'x.log'].forEach((file) => + writeFileSync(joinPath('ghost', file), 'content'), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply every rule except the one that ignores the pushed directory', async () => { + const paths = await getActorLocalFilePaths(joinPath('ghost')); + + expect(paths).toContain('main.js'); + expect(paths).toContain('src/index.js'); + expect(paths).not.toContain('secret.env'); + expect(paths).not.toContain('x.log'); + }); +}); + +const RANK_TEST_DIR = 'gitignore-rank-test-dir'; + +describe('Utils - a deeper .gitignore wins over the repository root one (no git)', () => { + // As in git, the `!` lines of an Actor's own .gitignore re-include what the root .gitignore ignores, except below a + // directory the root ignores. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(RANK_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git'), { recursive: true }); + mkdirSync(joinPath('actors', 'a', 'dist'), { recursive: true }); + mkdirSync(joinPath('actors', 'a', 'build'), { recursive: true }); + writeFileSync(joinPath('.gitignore'), 'dist/\nbuild/\n*.log\n*.map\n'); + writeFileSync(joinPath('actors', 'a', '.gitignore'), '!dist/\n!keep.log\n!build/keep.js\n'); + ['main.js', 'dist/bundle.js', 'dist/bundle.map', 'keep.log', 'other.log', 'build/keep.js'].forEach((file) => + writeFileSync(joinPath('actors', 'a', file), 'content'), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should include the files the deeper .gitignore re-includes', async () => { + const paths = await getActorLocalFilePaths(joinPath('actors', 'a')); + + expect(paths).toContain('main.js'); + expect(paths).toContain('dist/bundle.js'); + expect(paths).toContain('keep.log'); + expect(paths).not.toContain('other.log'); + expect(paths).not.toContain('dist/bundle.map'); + }); + + it('should keep a file below an ignored directory ignored', async () => { + const paths = await getActorLocalFilePaths(joinPath('actors', 'a')); + + expect(paths).not.toContain('build/keep.js'); + }); +}); + +const GIT_FILE_TEST_DIR = 'gitignore-git-file-test-dir'; + +describe('Utils - a .git file marks the repository root (no git)', () => { + // In a linked worktree or a submodule, .git is a file. The .gitignore above it belongs to an outer repository. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(GIT_FILE_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('worktree', 'actors', 'a'), { recursive: true }); + writeFileSync(joinPath('.gitignore'), 'outer.txt\n'); + writeFileSync(joinPath('worktree', '.git'), 'gitdir: ../elsewhere\n'); + writeFileSync(joinPath('worktree', '.gitignore'), '*.log\n'); + ['main.js', 'x.log', 'outer.txt'].forEach((file) => + writeFileSync(joinPath('worktree', 'actors', 'a', file), 'content'), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should stop at a .git file as at a .git directory', async () => { + const paths = await getActorLocalFilePaths(joinPath('worktree', 'actors', 'a')); + + expect(paths).toContain('main.js'); + expect(paths).toContain('outer.txt'); + expect(paths).not.toContain('x.log'); + }); +}); + +describe('Utils - no repository above the pushed directory (no git)', () => { + // Outside any repository, every .gitignore up to the filesystem root applies, scoped from there. test/tmp is inside + // the apify-cli repository, so the files go to the system temp directory. + let base: string; + + beforeAll(() => { + base = mkdtempSync(join(tmpdir(), 'apify-cli-no-repo-')); + mkdirSync(join(base, 'inner', 'build'), { recursive: true }); + writeFileSync(join(base, '.gitignore'), '*.secret\n/inner/build/\n'); + ['main.js', 'x.secret', 'build/out.js'].forEach((file) => writeFileSync(join(base, 'inner', file), 'content')); + }); + + afterAll(() => { + rmSync(base, { recursive: true, force: true }); + }); + + it('should apply every .gitignore above it', async () => { + const paths = await getActorLocalFilePaths(join(base, 'inner')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('x.secret'); + expect(paths).not.toContain('build/out.js'); + }); +}); + +const MIDDLE_TEST_DIR = 'gitignore-middle-test-dir'; + +describe('Utils - a .gitignore between the repository root and the pushed directory (no git)', () => { + // actors/.gitignore ignores the pushed actors/a with an anchored line. Its other lines apply from actors/, and rank + // above the root ones, as in git. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(MIDDLE_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + mkdirSync(joinPath('actors', 'a', 'local'), { recursive: true }); + writeFileSync(joinPath('.gitignore'), '*.log\n'); + writeFileSync(joinPath('actors', '.gitignore'), '/a/\na/local/\n!keep.log\n'); + ['main.js', 'keep.log', 'other.log', 'local/x.txt'].forEach((file) => + writeFileSync(joinPath('actors', 'a', file), 'content'), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should drop the line that ignores the pushed directory', async () => { + const paths = await getActorLocalFilePaths(joinPath('actors', 'a')); + + expect(paths).toContain('main.js'); + }); + + it('should anchor its lines to its own directory', async () => { + const paths = await getActorLocalFilePaths(joinPath('actors', 'a')); + + expect(paths).not.toContain('local/x.txt'); + }); + + it('should let its ! lines re-include what the root .gitignore ignores', async () => { + const paths = await getActorLocalFilePaths(joinPath('actors', 'a')); + + expect(paths).toContain('keep.log'); + expect(paths).not.toContain('other.log'); + }); +}); + +const RANK_INSIDE_TEST_DIR = 'gitignore-rank-inside-test-dir'; + +describe('Utils - a deeper .gitignore wins over a shallower one inside the pushed directory (no git)', () => { + // The example from the issue, pushed from the root: both files are inside the pushed directory, so their order + // decides. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(RANK_INSIDE_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + // '-' sorts before '.', so by name -sub/.gitignore would come before the root .gitignore. + ['sub', '-sub'].forEach((folder) => { + mkdirSync(joinPath(folder)); + writeFileSync(joinPath(folder, '.gitignore'), '!keep.log\n'); + }); + writeFileSync(joinPath('.gitignore'), '*.log\n'); + ['main.js', 'x.log', 'sub/keep.log', 'sub/other.log', '-sub/keep.log', '-sub/other.log'].forEach((file) => + writeFileSync(joinPath(file), 'content'), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should include the file the deeper .gitignore re-includes', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('main.js'); + expect(paths).toContain('sub/keep.log'); + expect(paths).not.toContain('x.log'); + expect(paths).not.toContain('sub/other.log'); + }); + + it('should rank the files by depth, not by name', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('-sub/keep.log'); + expect(paths).not.toContain('-sub/other.log'); + }); +}); + +const DOT_DIR_TEST_DIR = 'gitignore-dot-dir-test-dir'; + +describe('Utils - .gitignore inside a dot directory (no git)', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(DOT_DIR_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + mkdirSync(joinPath('.config')); + writeFileSync(joinPath('.config', '.gitignore'), 'secret.json\n'); + ['secret.json', 'settings.json'].forEach((file) => writeFileSync(joinPath('.config', file), 'content')); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply it', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('.config/settings.json'); + expect(paths).not.toContain('.config/secret.json'); + }); +}); + +const INSIDE_DOT_GIT_TEST_DIR = 'gitignore-inside-dot-git-test-dir'; + +describe('Utils - files inside .git (no git)', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(INSIDE_DOT_GIT_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git', 'hooks'), { recursive: true }); + writeFileSync(joinPath('.git', 'hooks', '.gitignore'), '*\n'); + writeFileSync(joinPath('main.js'), 'content'); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should not read a .gitignore inside .git', async () => { + vi.mocked(readFile).mockClear(); + + await getActorLocalFilePaths(tmpPath); + + const readPaths = vi.mocked(readFile).mock.calls.map(([path]) => String(path)); + expect(readPaths.filter((path) => path.includes(`${sep}.git${sep}`))).toEqual([]); + }); +}); + +const UNREADABLE_TEST_DIR = 'gitignore-unreadable-test-dir'; + +// chmod 000 has no effect on Windows and root reads the file anyway. +describe.skipIf(process.platform === 'win32' || process.getuid?.() === 0)( + 'Utils - an unreadable .gitignore inside the pushed directory (no git)', + () => { + // Git skips such a file with a warning. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(UNREADABLE_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + mkdirSync(joinPath('sub')); + writeFileSync(joinPath('sub', '.gitignore'), '*\n'); + chmodSync(joinPath('sub', '.gitignore'), 0o000); + ['main.js', 'sub/x.js'].forEach((file) => writeFileSync(joinPath(file), 'content')); + }); + + afterAll(async () => { + chmodSync(joinPath('sub', '.gitignore'), 0o644); + await afterAllCalls(); + }); + + it('should skip it', async () => { + await expect(getActorLocalFilePaths(tmpPath)).resolves.toEqual(expect.arrayContaining(['main.js', 'sub/x.js'])); + }); + }, +); + +const MISSING_TEST_DIR = 'gitignore-missing-test-dir'; + +describe('Utils - pushing a directory that does not exist (no git)', () => { + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(MISSING_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should find no files', async () => { + await expect(getActorLocalFilePaths(joinPath('missing'))).resolves.toEqual([]); + }); +}); + +const CRLF_TEST_DIR = 'gitignore-crlf-test-dir'; + +describe('Utils - .gitignore files with CRLF line endings (no git)', () => { + // The pushed directory is actors/a. actors/.gitignore is above it, actors/a/src/.gitignore inside it. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(CRLF_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + mkdirSync(joinPath('actors', 'a', 'src', 'cache'), { recursive: true }); + mkdirSync(joinPath('actors', 'a', 'src', 'lib', 'build'), { recursive: true }); + writeFileSync(joinPath('actors', '.gitignore'), '*.log\r\n\r\ncache/\r\n'); + writeFileSync(joinPath('actors', 'a', 'src', '.gitignore'), 'build/\r\n'); + ['main.js', 'src/index.js', 'x.log', 'src/cache/c.txt', 'src/lib/build/out.js'].forEach((file) => + writeFileSync(joinPath('actors', 'a', file), 'content'), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should read a .gitignore above the pushed directory as with LF line endings', async () => { + const paths = await getActorLocalFilePaths(joinPath('actors', 'a')); + + expect(paths).toContain('main.js'); + expect(paths).toContain('src/index.js'); + expect(paths).not.toContain('x.log'); + expect(paths).not.toContain('src/cache/c.txt'); + }); + + it('should read a .gitignore inside the pushed directory as with LF line endings', async () => { + const paths = await getActorLocalFilePaths(joinPath('actors', 'a')); + + expect(paths).not.toContain('src/lib/build/out.js'); + }); +}); + +const NFC_TEST_DIR = 'gitignore-nfc-test-dir'; + +describe('Utils - non-ASCII names in another Unicode form than the rules (no git)', () => { + // Git on macOS compares names in NFC. The rules and the names on disk can each be decomposed (NFD), as macOS + // archives store names. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(NFC_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + mkdirSync(joinPath('čaj'.normalize('NFD'), 'local'), { recursive: true }); + writeFileSync(joinPath('.gitignore'), `${'kůň.log'.normalize('NFD')}\nžába.log\n/čaj/local/\n`); + ['main.js', 'kůň.log', 'žába.log'.normalize('NFD')].forEach((file) => writeFileSync(joinPath(file), 'content')); + ['main.js', 'local/x.txt'].forEach((file) => writeFileSync(joinPath('čaj'.normalize('NFD'), file), 'content')); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should compare the rules and the paths in NFC', async () => { + const paths = (await getActorLocalFilePaths(tmpPath)).map((path) => path.normalize('NFC')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('kůň.log'); + expect(paths).not.toContain('žába.log'); + }); + + it('should compare the path of the pushed directory in NFC too', async () => { + const paths = await getActorLocalFilePaths(joinPath('čaj'.normalize('NFD'))); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('local/x.txt'); + }); +}); + +const NFC_TYPED_TEST_DIR = 'gitignore-nfc-typed-test-dir'; + +describe('Utils - pushing an ignored directory whose name is stored decomposed (no git)', () => { + // The directory is stored decomposed (NFD), as macOS archives store names. The push names it composed (NFC), as a + // keyboard types it, or as stored, as the current directory gives it. The line of the root .gitignore that ignores it + // is in NFC. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(NFC_TYPED_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + mkdirSync(joinPath('čaj'.normalize('NFD'))); + writeFileSync(joinPath('.gitignore'), `${'čaj'.normalize('NFC')}/\n*.log\n`); + ['main.js', 'x.log'].forEach((file) => writeFileSync(joinPath('čaj'.normalize('NFD'), file), 'content')); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should drop the line that ignores it and apply the rest, named composed', async ({ skip }) => { + const typed = joinPath('čaj'.normalize('NFC')); + // Only a file system that ignores the Unicode form of names, as macOS does, finds the directory by this path. + skip(!existsSync(typed), 'the file system tells the Unicode forms of names apart'); + + const paths = await getActorLocalFilePaths(typed); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('x.log'); + }); + + it('should drop the line that ignores it and apply the rest, named as stored', async () => { + const paths = await getActorLocalFilePaths(joinPath('čaj'.normalize('NFD'))); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('x.log'); + }); +}); + +const NFD_RULE_TEST_DIR = 'gitignore-nfd-rule-test-dir'; + +describe('Utils - pushing a directory a line written decomposed ignores (no git)', () => { + // The line of the root .gitignore is decomposed (NFD), as a file from a macOS archive may hold it, and the directory + // is stored and pushed composed (NFC). + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(NFD_RULE_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + mkdirSync(joinPath('čaj'.normalize('NFC'))); + writeFileSync(joinPath('.gitignore'), `${'čaj'.normalize('NFD')}/\n*.log\n`); + ['main.js', 'x.log'].forEach((file) => writeFileSync(joinPath('čaj'.normalize('NFC'), file), 'content')); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should drop the line that ignores it and apply the rest', async () => { + const paths = await getActorLocalFilePaths(joinPath('čaj'.normalize('NFC'))); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('x.log'); + }); +}); + +const PARTLY_IGNORED_TEST_DIR = 'gitignore-partly-ignored-test-dir'; + +describe('Utils - a .gitignore that ignores the pushed directory only in part (no git)', () => { + // `*` ignores a/, but `!*/` re-includes it, so the file doesn't ignore a/ and every line applies, as in git. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(PARTLY_IGNORED_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + mkdirSync(joinPath('a')); + writeFileSync(joinPath('.gitignore'), '*\n!*/\n!*.js\n'); + ['main.js', 'notes.txt'].forEach((file) => writeFileSync(joinPath('a', file), 'content')); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply every line', async () => { + const paths = await getActorLocalFilePaths(joinPath('a')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('notes.txt'); + }); +}); + +const ANCHOR_TEST_DIR = 'gitignore-anchor-test-dir'; + +describe('Utils - a nested .gitignore scopes its patterns as git does (no git)', () => { + // A pattern with a slash before its end matches only from sub/, one without matches at any depth below it. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(ANCHOR_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + ['sub/docs', 'sub/cache', 'sub/deep/docs', 'sub/deep/cache', 'sub2'].forEach((folder) => + mkdirSync(joinPath(folder), { recursive: true }), + ); + writeFileSync(joinPath('sub', '.gitignore'), '/top.txt\ndocs/draft.md\ncache/\n*.log\n'); + [ + 'sub/keep.js', + 'sub/top.txt', + 'sub/deep/top.txt', + 'sub/docs/draft.md', + 'sub/deep/docs/draft.md', + 'sub/cache/a.txt', + 'sub/deep/cache/b.txt', + 'sub/x.log', + 'sub/deep/y.log', + 'sub2/z.log', + ].forEach((file) => writeFileSync(joinPath(file), 'content')); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should anchor a pattern with a slash before its end to the directory of the .gitignore', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('sub/keep.js'); + expect(paths).not.toContain('sub/top.txt'); + expect(paths).toContain('sub/deep/top.txt'); + expect(paths).not.toContain('sub/docs/draft.md'); + expect(paths).toContain('sub/deep/docs/draft.md'); + }); + + it('should match a pattern without a slash before its end at any depth', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).not.toContain('sub/cache/a.txt'); + expect(paths).not.toContain('sub/deep/cache/b.txt'); + expect(paths).not.toContain('sub/x.log'); + expect(paths).not.toContain('sub/deep/y.log'); + }); + + it('should not apply its patterns to a sibling directory', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('sub2/z.log'); + }); +}); + +const LINES_TEST_DIR = 'gitignore-lines-test-dir'; + +describe('Utils - comments, blank lines and trailing spaces in a nested .gitignore (no git)', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(LINES_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + ['sub/build', 'sub/deep', 'other/deep/cache'].forEach((folder) => mkdirSync(joinPath(folder), { recursive: true })); + writeFileSync(joinPath('.gitignore'), 'build/\n*.txt\n'); + // A comment, a ! line whose pattern starts with #, a lone ! and a line of spaces. + writeFileSync(joinPath('sub', '.gitignore'), '#hash.md\n!#keep.txt\n!\n \n'); + // In a file of its own, so the line of spaces above doesn't hide it. + writeFileSync(joinPath('other', '.gitignore'), 'cache/ \n'); + [ + 'sub/#hash.md', + 'sub/#keep.txt', + 'sub/other.txt', + 'sub/build/out.js', + 'sub/deep/main.js', + 'other/deep/cache/b.js', + ].forEach((file) => writeFileSync(joinPath(file), 'content')); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should not treat a comment as a pattern', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('sub/#hash.md'); + }); + + it('should keep a ! line whose pattern starts with #', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('sub/#keep.txt'); + expect(paths).not.toContain('sub/other.txt'); + }); + + it('should ignore a lone !', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).not.toContain('sub/build/out.js'); + }); + + it('should ignore a line of spaces', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('sub/deep/main.js'); + }); + + it('should drop the trailing spaces of a pattern', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).not.toContain('other/deep/cache/b.js'); + }); +}); + +const ESCAPED_SPACE_TEST_DIR = 'gitignore-escaped-space-test-dir'; + +// Windows doesn't allow a file name that ends with a space. +describe.skipIf(process.platform === 'win32')( + 'Utils - an escaped trailing space in a nested .gitignore (no git)', + () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(ESCAPED_SPACE_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + mkdirSync(joinPath('sub')); + writeFileSync(joinPath('sub', '.gitignore'), 'name\\ \n'); + ['sub/name ', 'sub/name'].forEach((file) => writeFileSync(joinPath(file), 'content')); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should keep the escaped space', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).not.toContain('sub/name '); + expect(paths).toContain('sub/name'); + }); + }, +); + +const PATTERN_DIR_NAMES_TEST_DIR = 'gitignore-pattern-dir-names-test-dir'; + +describe('Utils - .gitignore files in directories whose names look like patterns (no git)', () => { + // The directory name starts each scoped line, so its [ and a leading ! or # must match literally. + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(PATTERN_DIR_NAMES_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + ['[ab]', 'a', '!neg/inner', '#hash'].forEach((folder) => mkdirSync(joinPath(folder), { recursive: true })); + ['[ab]', '!neg', '#hash'].forEach((folder) => writeFileSync(joinPath(folder, '.gitignore'), '*.log\n')); + ['[ab]/x.log', '[ab]/main.js', 'a/y.log', '!neg/x.log', '!neg/inner/z.log', '#hash/x.log'].forEach((file) => + writeFileSync(joinPath(file), 'content'), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply the rules to that directory only', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).toContain('[ab]/main.js'); + expect(paths).not.toContain('[ab]/x.log'); + expect(paths).toContain('a/y.log'); + expect(paths).not.toContain('!neg/x.log'); + expect(paths).not.toContain('#hash/x.log'); + }); + + it('should apply the rules of such a directory above the pushed one', async () => { + const paths = await getActorLocalFilePaths(joinPath('!neg', 'inner')); + + expect(paths).not.toContain('z.log'); + }); +}); + +const GLOB_DIR_NAMES_TEST_DIR = 'gitignore-glob-dir-names-test-dir'; + +// Windows doesn't allow *, ? or \ in a file name. +describe.skipIf(process.platform === 'win32')( + 'Utils - .gitignore files in directories whose names hold *, ? or \\ (no git)', + () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath(GLOB_DIR_NAMES_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('.git')); + ['a*b', 'aXb', 'c?d', 'cXd', 'e\\f', 'a*b*c', 'a*bXc'].forEach((folder) => mkdirSync(joinPath(folder))); + ['a*b', 'c?d', 'e\\f', 'a*b*c'].forEach((folder) => writeFileSync(joinPath(folder, '.gitignore'), '*.log\n')); + ['a*b/x.log', 'aXb/y.log', 'c?d/x.log', 'cXd/y.log', 'e\\f/x.log', 'a*b*c/x.log', 'a*bXc/y.log'].forEach((file) => + writeFileSync(joinPath(file), 'content'), + ); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply the rules to that directory only', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).not.toContain('a*b/x.log'); + expect(paths).toContain('aXb/y.log'); + expect(paths).not.toContain('c?d/x.log'); + expect(paths).toContain('cXd/y.log'); + expect(paths).not.toContain('e\\f/x.log'); + }); + + it('should match every glob character of the name literally', async () => { + const paths = await getActorLocalFilePaths(tmpPath); + + expect(paths).not.toContain('a*b*c/x.log'); + expect(paths).toContain('a*bXc/y.log'); + }); + }, +); + +const NO_EXCLUDES_TEST_DIR = 'gitignore-no-excludes-test-dir'; + +describe('Utils - exclude files when git is unavailable (no git)', () => { + // Git names .git/info/exclude and the global excludes file, so without git only the .gitignore files apply. + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath(NO_EXCLUDES_TEST_DIR, { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + + mkdirSync(joinPath('repo', '.git', 'info'), { recursive: true }); + mkdirSync(joinPath('home', '.config', 'git'), { recursive: true }); + mkdirSync(joinPath('xdg', 'git'), { recursive: true }); + writeFileSync(joinPath('repo', '.git', 'info', 'exclude'), '*.pem\n'); + writeFileSync(joinPath('home', '.config', 'git', 'ignore'), '*.bak\n'); + writeFileSync(joinPath('xdg', 'git', 'ignore'), '*.bak\n'); + writeFileSync(joinPath('repo', '.gitignore'), '*.log\n'); + ['main.js', 'x.log', 'key.pem', 'notes.bak'].forEach((file) => writeFileSync(joinPath('repo', file), 'content')); + }); + + afterEach(() => { + vitest.unstubAllEnvs(); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('should apply neither .git/info/exclude nor the global excludes file', async () => { + // os.homedir() reads USERPROFILE on Windows and HOME elsewhere. + vitest.stubEnv('HOME', joinPath('home')); + vitest.stubEnv('USERPROFILE', joinPath('home')); + vitest.stubEnv('XDG_CONFIG_HOME', joinPath('xdg')); + + const paths = await getActorLocalFilePaths(joinPath('repo')); + + expect(paths).toContain('main.js'); + expect(paths).not.toContain('x.log'); + expect(paths).toContain('key.pem'); + expect(paths).toContain('notes.bak'); + }); +});