diff --git a/docs/reference.md b/docs/reference.md index 420618be4..329bec2b0 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -859,6 +859,10 @@ DESCRIPTION as ZIP file. Files matched by .gitignore and .actorignore are excluded. Use negation patterns (e.g. !dist/) in .actorignore to force-include git-ignored files. + If "dockerContextDir" leads outside the Actor folder (a monorepo Actor), push + uploads the whole Git repository: the Actor folder as above, and everything + else that Git tracks or doesn't ignore (.actorignore doesn't apply there). + Outside Git, it uploads the whole Docker context as above. Use --force to override newer remote versions. USAGE diff --git a/src/commands/actors/push.ts b/src/commands/actors/push.ts index b4f5f3080..68facb989 100644 --- a/src/commands/actors/push.ts +++ b/src/commands/actors/push.ts @@ -1,5 +1,7 @@ -import { readFileSync, statSync, unlinkSync, writeSync } from 'node:fs'; -import { basename, join, resolve } from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; +import { lstatSync, readFileSync, realpathSync, rmSync, type Stats, statSync, writeSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, relative, resolve } from 'node:path'; import process from 'node:process'; import type { Actor, ActorCollectionCreateOptions, ActorDefaultRunOptions, Build } from 'apify-client'; @@ -23,7 +25,14 @@ import { INTERRUPT_SIGNALS, LOCAL_CONFIG_PATH, } from '../../lib/consts.js'; -import { monorepoUnsupportedMessage, resolveDockerContext } from '../../lib/docker-context.js'; +import { + getContextFilePaths, + isOutside, + readGitState, + realPath, + resolveDockerContext, + toPosix, +} from '../../lib/docker-context.js'; import { sumFilesSizeInBytes } from '../../lib/files.js'; import { findActorDirs, type FoundActorDir, toPosixRelative } from '../../lib/find-actor-dirs.js'; import { isAbortingJobOnSignal, useAbortJobOnSignal } from '../../lib/hooks/useAbortJobOnSignal.js'; @@ -45,13 +54,14 @@ import { printJsonToStdout, } from '../../lib/utils.js'; -const TEMP_ZIP_FILE_NAME = 'temp_file.zip'; const DEFAULT_RUN_OPTIONS = { build: 'latest', memoryMbytes: 4096, timeoutSecs: 3600, }; const DEFAULT_ACTOR_VERSION_NUMBER = '0.0'; +/** The longest name of a multi-file source file the platform accepts. */ +const MAX_SOURCE_FILE_NAME_LENGTH = 255; // It would be better to use `version-0.0` or similar, // or even have no default tag, but the platform complains when @@ -205,6 +215,199 @@ const confirmGitSourceSwitch = async ({ }); }; +/** + * Push sends files only, in the multi-file and the zip upload alike: a link to a file the upload holds goes as that + * file at the link's path, a link to a directory inside the upload as copies of the uploaded files under its target + * (directory links there included), and a link to nothing, to a file the upload leaves out (such as a gitignored + * .env), or to a target outside the upload is left out, as a clone of the repository could not follow it either. A + * link to a file that only the Actor folder's upload adds, such as a gitignored dist that its .actorignore takes back, + * goes too, and so does a directory link to such files. The `actor.json` path check then reports a config that refers + * to a left-out link as not uploaded. + */ +const resolveLinksForUpload = (root: string, files: string[], actorPath: string, actorFolderFiles: string[]) => { + const realRoot = realpathSync(root); + // Git lists names composed (NFC) on macOS, while a link's target keeps the form its text has. + const listed = new Set(files.map((file) => file.normalize('NFC'))); + // The Actor folder's files have their names as stored on disk, as a target's native real path spells them. + const actorFolderUpload = new Set(actorFolderFiles); + // Outside Git, the file list follows folder links, so a file that is no link itself can lie outside the upload too. + // Its folder's real path tells, once per folder. + const folders = new Map(); + const isFolderInUpload = (folder: string) => { + let inUpload = folders.get(folder); + if (inUpload === undefined) { + try { + inUpload = !isOutside(relative(realRoot, realpathSync(join(root, folder)))); + } catch { + inUpload = false; + } + folders.set(folder, inUpload); + } + return inUpload; + }; + // Where an entry's content lies, relative to the root, as the link spells it and as stored on disk, and what it is; + // nothing for a link the upload drops. + interface Located { + target: string; + diskTarget: string; + stats: Stats; + } + const locate = (file: string): Located | undefined => { + const filePath = join(root, file); + const stats = lstatSync(filePath, { throwIfNoEntry: false }); + if (!stats) return undefined; + if (!stats.isSymbolicLink()) { + return isFolderInUpload(dirname(file)) ? { target: file, diskTarget: file, stats } : undefined; + } + try { + const realFilePath = realpathSync(filePath); + const target = relative(realRoot, realFilePath); + if (isOutside(target)) return undefined; + const targetStats = statSync(realFilePath); + const diskTarget = toPosix(relative(realRoot, realPath(filePath))); + if (targetStats.isFile() && !listed.has(toPosix(target).normalize('NFC')) && !actorFolderUpload.has(diskTarget)) { + return undefined; + } + return { target: toPosix(target), diskTarget, stats: targetStats }; + } catch { + return undefined; + } + }; + const located = new Map(files.map((file) => [file, locate(file)])); + // Git may spell the Actor folder otherwise than the disk stores it: composed (NFC) on macOS, or in its old case after + // a case-only rename. Its spelling in Git's list is confirmed on disk, as that of another folder can match too. + const fold = (name: string) => name.normalize('NFC').toLowerCase(); + const actorDepth = actorPath.split('/').length; + const gitActorPath = + [ + ...new Set( + files + .filter((file) => fold(file).startsWith(fold(`${actorPath}/`))) + .map((file) => file.split('/').slice(0, actorDepth).join('/')), + ), + ].find((folder) => { + try { + return realPath(join(root, folder)) === join(realRoot, actorPath); + } catch { + return false; + } + }) ?? actorPath; + // The Actor folder's upload adds files a clone lacks, such as its gitignored dist, which a directory link copies too. + // A file Git lists in another spelling is no such file, which the disk confirms, as on a case-sensitive file system + // names that differ in case only are two files. Each maps to its name with the Actor folder as Git spells it. + const listedByFold = new Map(files.map((file) => [fold(file), file])); + const isListed = (file: string) => { + if (listed.has(file.normalize('NFC'))) return true; + const spelling = listedByFold.get(fold(file)); + if (spelling === undefined) return false; + try { + return realPath(join(root, spelling)) === realPath(join(root, file)); + } catch { + return false; + } + }; + const actorFolderOnly = new Map( + actorFolderFiles + .filter((file) => !isListed(file)) + .map((file) => [file, `${gitActorPath}${file.slice(actorPath.length)}`]), + ); + for (const file of actorFolderOnly.keys()) { + let entry: Located | undefined; + try { + const realFilePath = realPath(join(root, file)); + const stats = statSync(realFilePath); + if (stats.isFile() && !isOutside(relative(realRoot, realFilePath))) + entry = { target: file, diskTarget: file, stats }; + } catch { + // A file that vanished since it was listed goes nowhere. + } + located.set(file, entry); + } + const uploaded = [...files, ...actorFolderOnly.keys()]; + // A directory link copies the files under its target, and through a directory link there, the files under that + // one's target, as a clone resolves them; a link back to a target on the way copies nothing, which ends a cycle. + // The Actor folder's own files match the target as Git spells the Actor folder, so their copies sit beside those of + // its listed files, else as the disk stores it. + const expand = (file: string, targets: string[]): string[] => { + const entry = located.get(file); + if (!entry) return []; + if (entry.stats.isFile()) return [file]; + if (!entry.stats.isDirectory() || targets.includes(entry.target)) return []; + return uploaded.flatMap((other) => { + const gitSpelling = actorFolderOnly.get(other); + const [name, prefix] = + gitSpelling === undefined || gitSpelling.startsWith(`${entry.target}/`) + ? [gitSpelling ?? other, `${entry.target}/`] + : [other, `${entry.diskTarget}/`]; + if (!name.startsWith(prefix)) return []; + const copyPath = `${file}/${name.slice(prefix.length)}`; + return expand(other, [...targets, entry.target]).map((copy) => `${copyPath}${copy.slice(other.length)}`); + }); + }; + return files.flatMap((file) => expand(file, [])); +}; + +/** + * The Actor folder goes as an ordinary push of it would send it: its .gitignore and .actorignore apply, negations + * included, and an ignored folder that was picked or named with --dir goes too. Only a file whose link leads out of + * the upload stays out, as in the rest of the upload. + */ +const withActorFolderFiles = (root: string, actorPath: string, contextFiles: string[], actorFiles: string[]) => { + // The root has its names in their case on disk, so a link that spells a folder in another case must resolve so too. + const realRoot = realPath(root); + const isInUpload = (file: string) => { + try { + return !isOutside(relative(realRoot, realPath(join(root, file)))); + } catch { + return false; + } + }; + // Git lists names as its index has them: composed (NFC) on macOS, and in the old case after a case-only rename on a + // case-insensitive file system. actorPath has the names as stored on disk, so a folder on its way matches when its + // name differs only so and it is the same folder on disk, checked once for each spelling. + const actorSegments = actorPath.split('/'); + const fold = (name: string) => name.normalize('NFC').toLowerCase(); + const spellings = new Map(); + const isActorFolderOnTheWay = (segments: string[], depth: number) => { + const folder = segments.slice(0, depth).join('/'); + let matches = spellings.get(folder); + if (matches === undefined) { + try { + // A link named like the folder in another case, on a case-sensitive file system, is a folder of its own. + matches = + !lstatSync(join(root, folder)).isSymbolicLink() && + realPath(join(root, folder)) === join(realRoot, ...actorSegments.slice(0, depth)); + } catch { + matches = false; + } + spellings.set(folder, matches); + } + return matches; + }; + const contextFilesOutsideActorFolder = contextFiles.flatMap((file) => { + const segments = file.split('/'); + let depth = 0; + while ( + depth < actorSegments.length && + depth < segments.length - 1 && + fold(segments[depth]) === fold(actorSegments[depth]) && + isActorFolderOnTheWay(segments, depth + 1) + ) { + depth++; + } + if (depth === actorSegments.length) return []; + // A folder on the Actor folder's way goes under one name, as actorPath spells it, or the platform would write + // the Actor folder and its siblings into two folders. + return [[...actorSegments.slice(0, depth), ...segments.slice(depth)].join('/')]; + }); + return [ + ...new Set([ + ...contextFilesOutsideActorFolder, + ...actorFiles.map((file) => `${actorPath}/${file}`).filter(isInUpload), + ]), + ]; +}; + /** The config files that make a directory an Actor of its own: `.actor/actor.json`, `actor.json`, or the legacy `apify.json`. */ const OWN_CONFIG_FILES = [DEPRECATED_LOCAL_CONFIG_NAME, 'actor.json', '.actor/actor.json']; @@ -233,6 +436,9 @@ export class ActorsPushCommand extends ApifyCommand { `larger projects upload as ZIP file.\n` + `Files matched by .gitignore and .actorignore are excluded. ` + `Use negation patterns (e.g. !dist/) in .actorignore to force-include git-ignored files.\n` + + `If "dockerContextDir" leads outside the Actor folder (a monorepo Actor), push uploads the whole Git repository: ` + + `the Actor folder as above, and everything else that Git tracks or doesn't ignore (.actorignore doesn't apply there). ` + + `Outside Git, it uploads the whole Docker context as above.\n` + `Use --force to override newer remote versions.`; static override group = 'Local Actor Development'; @@ -538,27 +744,42 @@ export class ActorsPushCommand extends ApifyCommand { const { config: actorConfig } = actorConfigResult.unwrap(); - // A monorepo Actor builds from a Docker context above its folder, which a push of that folder cannot carry. + // A monorepo Actor builds from a Docker context above its folder, so the push uploads its repository + // (or, outside Git, the context) and tells the platform where the Actor lies in it. const dockerContext = resolveDockerContext(cwd, actorConfig?.dockerContextDir); if (dockerContext.kind === 'invalid') { error({ message: dockerContext.message }); process.exitCode = CommandExitCodes.InvalidActorJson; return; } + // A context inside the Actor folder ('none' or 'within') is an ordinary push of the folder. + let actorDir = cwd; + let uploadRoot = cwd; + let uploadedFiles = filePathsToPush; + let actorPath: string | undefined; if (dockerContext.kind === 'outside') { - // Pushing by ID, actor.json may have no name; the folder then names the Actor. - error({ message: monorepoUnsupportedMessage((actorConfig?.name as string | undefined) ?? basename(cwd)) }); - process.exitCode = CommandExitCodes.NotImplemented; - return; + uploadRoot = dockerContext.sourceRoot; + actorPath = dockerContext.actorPath; + // The source root is a real path, so the Actor folder has to be one too for its paths to fall inside it. + actorDir = join(uploadRoot, actorPath); + const actorFolderFiles = filePathsToPush.map((file) => `${actorPath}/${file}`); + const contextFiles = resolveLinksForUpload( + uploadRoot, + await getContextFilePaths(uploadRoot), + actorPath, + actorFolderFiles, + ); + uploadedFiles = withActorFolderFiles(uploadRoot, actorPath, contextFiles, filePathsToPush); } // The platform resolves these paths after the upload; a wrong one builds the wrong image or fails the build. const pathProblems = checkActorPaths({ - actorDir: cwd, - uploadRoot: cwd, - uploadedFiles: filePathsToPush, + actorDir, + uploadRoot, + uploadedFiles, config: actorConfig, - dockerContextRoot: dockerContext.kind === 'within' ? dockerContext.contextRoot : undefined, + // BuildKit reads the Dockerfile from the context, wherever the context lies. + dockerContextRoot: dockerContext.kind === 'none' ? undefined : dockerContext.contextRoot, }); // A readme or changelog the platform cannot find only falls back to its default, so push goes on. const pathWarnings = formatActorPathProblems(pathProblems, 'warning'); @@ -570,6 +791,12 @@ export class ActorsPushCommand extends ApifyCommand { return; } + if (actorPath && readGitState(uploadRoot).dirty) { + warning({ + message: `The repository ${uploadRoot} has uncommitted changes; the push uploads the files as they are.`, + }); + } + const apifyClient = await getLoggedClientOrThrow(); const userInfo = await getCurrentUserInfo(); const isOrganizationLoggedIn = !!userInfo.organizationOwnerUserId; @@ -623,6 +850,7 @@ export class ActorsPushCommand extends ApifyCommand { // TODO: export enum from apify-client sourceType: ACTOR_SOURCE_TYPES.SOURCE_FILES as never, sourceFiles: [], + ...(actorPath ? { actorPath } : {}), }, ], }; @@ -648,14 +876,14 @@ export class ActorsPushCommand extends ApifyCommand { info({ message: `Deploying Actor '${actorConfig!.name}' to Apify.` }); - const filesSize = await sumFilesSizeInBytes(filePathsToPush, cwd); + const filesSize = await sumFilesSizeInBytes(uploadedFiles, uploadRoot); - if (filesSize < MAX_MULTIFILE_BYTES && !isActorCreatedNow) { + if ((filesSize < MAX_MULTIFILE_BYTES || actorPath) && !isActorCreatedNow) { const client = await actorClient.get(); // Check when was files modified last - const mostRecentModifiedFileMs = filePathsToPush.reduce((modifiedMs, filePath) => { - const { mtimeMs, ctimeMs } = statSync(join(cwd, filePath)); + const mostRecentModifiedFileMs = uploadedFiles.reduce((modifiedMs, filePath) => { + const { mtimeMs, ctimeMs } = statSync(join(uploadRoot, filePath)); // Sometimes it's possible mtimeMs is some messed up value (like 2000/01/01 midnight), then we want to check created if it's newer const fileModifiedMs = mtimeMs > ctimeMs ? mtimeMs : ctimeMs; @@ -695,27 +923,50 @@ Skipping push. Use --force to override.`, } let sourceType; - let sourceFiles; let tarballUrl; - if (filesSize < MAX_MULTIFILE_BYTES) { - sourceFiles = await createSourceFiles(filePathsToPush, cwd); + // The platform rejects a source file name over its limit, which a deep path from the repository root can reach, + // and measures the content as sent, where a binary file grows by a third in base64 and each character beyond the + // Basic Multilingual Plane, such as an emoji, counts five bytes; a zip has neither limit. + let sourceFiles = + filesSize < MAX_MULTIFILE_BYTES && uploadedFiles.every((file) => file.length <= MAX_SOURCE_FILE_NAME_LENGTH) + ? await createSourceFiles(uploadedFiles, uploadRoot) + : undefined; + if ( + sourceFiles && + sourceFiles.reduce( + (size, file) => size + Buffer.byteLength(file.content) + (file.content.match(/[\uD800-\uDBFF]/g)?.length ?? 0), + 0, + ) > MAX_MULTIFILE_BYTES + ) { + sourceFiles = undefined; + } + if (sourceFiles) { sourceType = ACTOR_SOURCE_TYPES.SOURCE_FILES; } else { // Create zip run({ message: 'Zipping Actor files' }); - await createActZip(TEMP_ZIP_FILE_NAME, filePathsToPush, cwd); - - // Upload it to Apify.keyValueStores const store = await apifyClient.keyValueStores().getOrCreate(`actor-${actorId}-source`); - const key = `version-${version}.zip`; - const buffer = readFileSync(TEMP_ZIP_FILE_NAME); - await apifyClient.keyValueStore(store.id).setRecord({ - key, - // TODO: fix this type too - value: buffer as never, - contentType: 'application/zip', - }); - unlinkSync(TEMP_ZIP_FILE_NAME); + // A monorepo zip has the repository layout, so it gets a key of its own for each actorPath: when the version + // update then fails, the existing version still builds from its own record, whether it has no actorPath or + // another one, such as the Actor folder's name before a rename. + const key = actorPath + ? `version-${version}-monorepo-${createHash('sha256').update(actorPath).digest('hex').slice(0, 16)}.zip` + : `version-${version}.zip`; + const zipPath = join(tmpdir(), `apify-push-${randomUUID()}.zip`); + try { + await createActZip(zipPath, uploadedFiles, uploadRoot); + + // Upload it to Apify.keyValueStores + const buffer = readFileSync(zipPath); + await apifyClient.keyValueStore(store.id).setRecord({ + key, + // TODO: fix this type too + value: buffer as never, + contentType: 'application/zip', + }); + } finally { + rmSync(zipPath, { force: true }); + } const tempTarballUrl = new URL( `${apifyClient.baseUrl}/key-value-stores/${store.id}/records/${key}?disableRedirect=true`, ); @@ -748,18 +999,17 @@ Skipping push. Use --force to override.`, const applyEnvVarsToBuild = this.flags.applyEnvVarsToBuild ?? (actorConfig!.applyEnvVarsToBuild as boolean | undefined); + const actorVersionSource = { tarballUrl, sourceFiles, sourceType, ...(actorPath ? { actorPath } : {}) }; if (actorCurrentVersion) { - const actorVersionModifier = { tarballUrl, sourceFiles, buildTag, sourceType, envVars, applyEnvVarsToBuild }; + const actorVersionModifier = { ...actorVersionSource, buildTag, envVars, applyEnvVarsToBuild }; // TODO: fix this type too -.- await actorClient.version(version).update(actorVersionModifier as never); run({ message: `Updated version ${version} for Actor ${actor.name}.` }); } else { const actorNewVersion = { versionNumber: version, - tarballUrl, - sourceFiles, + ...actorVersionSource, buildTag, - sourceType, envVars, applyEnvVarsToBuild, }; diff --git a/src/lib/actor-paths.ts b/src/lib/actor-paths.ts index 721a405ae..4482208bf 100644 --- a/src/lib/actor-paths.ts +++ b/src/lib/actor-paths.ts @@ -1,8 +1,8 @@ -import { readdirSync, realpathSync, type Stats, statSync } from 'node:fs'; +import { readdirSync, type Stats, statSync } from 'node:fs'; import { basename, dirname, isAbsolute, join, posix, relative, resolve } from 'node:path'; import { ACTOR_SPECIFICATION_FOLDER } from './consts.js'; -import { isOutside, toPosix } from './docker-context.js'; +import { isOutside, realPath, toPosix } from './docker-context.js'; /** * An `error` would fail the build or build the wrong thing, so push stops. A `warning` is a path the platform cannot @@ -248,12 +248,12 @@ function checkDefaultDockerfile(scope: PathCheckScope & { dockerContextRoot: str * as push uploads its content there. With a Docker context and no `dockerfile`, the default Dockerfile is checked too. */ export function checkActorPaths(options: ActorPathsOptions): ActorPathProblem[] { - // The Actor folder and the upload root are real paths, as the Docker context resolver works from them, so a - // symlinked Actor folder does not count as an escape. A referenced path, like the Docker context the resolver - // returns, is judged as written below the real Actor folder, the way push lays out the upload. + // The Actor folder and the upload root are real paths, in their case on disk, as the Docker context resolver works + // from them, so a symlinked Actor folder does not count as an escape. A referenced path, like the Docker context the + // resolver returns, is judged as written below the real Actor folder, the way push lays out the upload. const scope: PathCheckScope = { - actorDir: realpathSync(options.actorDir), - uploadRoot: realpathSync(options.uploadRoot), + actorDir: realPath(options.actorDir), + uploadRoot: realPath(options.uploadRoot), uploadedFiles: new Set(options.uploadedFiles.map(toPosix)), dockerContextRoot: options.dockerContextRoot, }; diff --git a/src/lib/docker-context.ts b/src/lib/docker-context.ts index 47ee7abe4..0a14f8e8f 100644 --- a/src/lib/docker-context.ts +++ b/src/lib/docker-context.ts @@ -1,6 +1,5 @@ // The Docker context of an Actor: where `dockerContextDir` in `.actor/actor.json` points and what a push of it -// would carry. `readGitState` and `getContextFilePaths` have no caller in `apify push` yet; they serve the -// stage 2 upload of the context and the `actor-runtime` branch. +// carries. import { execFileSync } from 'node:child_process'; import { lstatSync, realpathSync, statSync } from 'node:fs'; import { isAbsolute, join, relative, resolve, sep } from 'node:path'; @@ -8,9 +7,6 @@ import { isAbsolute, join, relative, resolve, sep } from 'node:path'; import { ACTOR_SPECIFICATION_FOLDER } from './consts.js'; import { getActorLocalFilePaths } from './utils.js'; -/** Tracks pushing Actors with `dockerContextDir` (monorepo Actors) to the Apify platform. */ -export const MONOREPO_PLATFORM_SUPPORT_ISSUE_URL = 'https://github.com/apify/apify-core/issues/28685'; - export type DockerContextResolution = /** `dockerContextDir` is absent: undefined, null, or an empty string. */ | { kind: 'none' } @@ -27,7 +23,7 @@ export type DockerContextResolution = contextRoot: string; /** The Git repository's root, else the context. */ sourceRoot: string; - /** Relative to `sourceRoot`. */ + /** Relative to `sourceRoot`, never empty, with the names as stored on disk. */ actorPath: string; } /** The field is not a string, or the directory it names is missing, not a directory, or out of reach. */ @@ -45,11 +41,27 @@ export function resolveDockerContext(actorDir: string, dockerContextDir: unknown if (typeof dockerContextDir !== 'string') { return { kind: 'invalid', message: `"dockerContextDir" in .actor/actor.json must be a string.` }; } + // The platform resolves the path from .actor/ on Linux, as the paths to files in actor.json. + if (isAbsolute(dockerContextDir)) { + return { + kind: 'invalid', + message: `"dockerContextDir" in .actor/actor.json is absolute (${dockerContextDir}); the path is relative to .actor/.`, + }; + } + if (dockerContextDir.includes('\\')) { + return { + kind: 'invalid', + message: + `"dockerContextDir" in .actor/actor.json uses a backslash (${dockerContextDir}); ` + + 'the platform resolves paths with forward slashes.', + }; + } - // The platform resolves the context from the Actor folder's real path, so a symlinked `--dir` must too. + // The platform resolves the context from the Actor folder's real path, so a symlinked `--dir` must too. The native + // call also gives the names their case on disk, which a `--dir` typed in another case on macOS or Windows lacks. let realActorDir: string; try { - realActorDir = realpathSync(actorDir); + realActorDir = realPath(actorDir); } catch { return { kind: 'invalid', message: `The Actor folder ${actorDir} does not exist.` }; } @@ -68,7 +80,7 @@ export function resolveDockerContext(actorDir: string, dockerContextDir: unknown message: `"dockerContextDir" in .actor/actor.json points to ${contextRoot}, which is not a directory.`, }; } - const realContextRoot = realpathSync(contextRoot); + const realContextRoot = realPath(contextRoot); // Inside or outside the Actor folder is judged, and returned, by the path as written, not by where a symlink on it // leads. Push follows a symlink and uploads its files under the link's own name: with `"dockerContextDir": "../sym"` @@ -81,8 +93,9 @@ export function resolveDockerContext(actorDir: string, dockerContextDir: unknown // The platform's clone holds nothing above the repository root. const repoRoot = git(realActorDir, ['rev-parse', '--show-toplevel']); + let sourceRoot = realContextRoot; if (repoRoot) { - const sourceRoot = realpathSync(repoRoot); + sourceRoot = realPath(repoRoot); if (isOutside(relative(sourceRoot, realContextRoot))) { return { kind: 'invalid', @@ -91,22 +104,34 @@ export function resolveDockerContext(actorDir: string, dockerContextDir: unknown 'The Apify platform builds from a clone of the repository, so it could not build this Actor.', }; } - return { - kind: 'outside', - contextRoot, - sourceRoot, - actorPath: toPosix(relative(sourceRoot, realActorDir)), - }; } - const actorPath = relative(realContextRoot, realActorDir); + const actorPath = relative(sourceRoot, realActorDir); if (isOutside(actorPath)) { return { kind: 'invalid', message: `"dockerContextDir" in .actor/actor.json points to ${realContextRoot}, which does not contain the Actor's folder.`, }; } - return { kind: 'outside', contextRoot, sourceRoot: realContextRoot, actorPath: toPosix(actorPath) }; + // The path as written leaves the Actor folder, so it isn't an ordinary push, yet the upload root would be the folder. + if (actorPath === '') { + return { + kind: 'invalid', + message: + `"dockerContextDir" in .actor/actor.json leaves the Actor folder, then leads back into it at ${realContextRoot}, ` + + 'through a symlink or a folder name in another letter case. Write the path without leaving the Actor folder.', + }; + } + return { kind: 'outside', contextRoot, sourceRoot, actorPath: toPosix(actorPath) }; +} + +/** The real path with names in their case on disk; the JS implementation where the native call fails, as on Windows RAM or virtual disks. */ +export function realPath(path: string): string { + try { + return realpathSync.native(path); + } catch { + return realpathSync(path); + } } export function isOutside(relativePath: string): boolean { @@ -126,8 +151,15 @@ export interface GitState { function git(cwd: string, args: string[]): string | undefined { try { - const output = execFileSync('git', args, { cwd, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }); - return output.trim(); + // A large repository lists more than the default 1 MiB of file names. + const output = execFileSync('git', args, { + cwd, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + maxBuffer: Number.POSITIVE_INFINITY, + }); + // Only the line end goes: a file or folder name can start or end with a space. + return output.replace(/\r?\n$/, ''); } catch { return undefined; } @@ -148,14 +180,15 @@ function withoutCredentials(remoteUrl: string): string { export function readGitState(dir: string): GitState { const commit = git(dir, ['rev-parse', 'HEAD']); - if (!commit) return {}; + const status = git(dir, ['status', '--porcelain', '--', '.']); + // Before the first commit, every file is uncommitted. + if (!commit) return status ? { dirty: true } : {}; const state: GitState = { commit }; const branch = git(dir, ['rev-parse', '--abbrev-ref', 'HEAD']); if (branch && branch !== 'HEAD') state.branch = branch; const remoteUrl = git(dir, ['remote', 'get-url', 'origin']); if (remoteUrl) state.remoteUrl = withoutCredentials(remoteUrl); - const status = git(dir, ['status', '--porcelain', '--', '.']); if (status !== undefined) state.dirty = status.length > 0; return state; } @@ -165,7 +198,8 @@ const UNTRACKED_IGNORED_SEGMENTS = new Set(['node_modules', 'storage', 'apify_st function gitFileList(root: string, args: string[]): string[] | undefined { const output = git(root, ['ls-files', '-z', ...args]); - return output?.split('\0').filter(Boolean); + // During an unresolved merge, --cached lists a conflicted path once per index stage. + return output === undefined ? undefined : [...new Set(output.split('\0').filter(Boolean))]; } /** @@ -195,12 +229,3 @@ export async function getContextFilePaths(root: string): Promise { ), ]; } - -export function monorepoUnsupportedMessage(actorName: string): string { - return [ - `Actor ${actorName} sets "dockerContextDir" in .actor/actor.json, so it builds from a Docker context outside its own folder (a monorepo Actor).`, - 'apify push does not support such Actors on the Apify platform yet. Follow the progress at:', - ` ${MONOREPO_PLATFORM_SUPPORT_ISSUE_URL}`, - `Until then, set the Actor's source in Apify Console to the Git repository with the folder (#:).`, - ].join('\n'); -} diff --git a/test/local/commands/push-monorepo.test.ts b/test/local/commands/push-monorepo.test.ts index 7aef37d4d..b4214d23e 100644 --- a/test/local/commands/push-monorepo.test.ts +++ b/test/local/commands/push-monorepo.test.ts @@ -1,12 +1,16 @@ import { execFileSync } from 'node:child_process'; +import { existsSync, readdirSync, renameSync, symlinkSync } from 'node:fs'; import { mkdir, writeFile } from 'node:fs/promises'; -import { dirname } from 'node:path'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; -import { ACTOR_SOURCE_TYPES } from '@apify/consts'; +import AdmZip from 'adm-zip'; + +import { ACTOR_SOURCE_TYPES, MAX_MULTIFILE_BYTES } from '@apify/consts'; import { testRunCommand } from '../../../src/lib/command-framework/apify-command.js'; import { CommandExitCodes } from '../../../src/lib/consts.js'; -import { MONOREPO_PLATFORM_SUPPORT_ISSUE_URL } from '../../../src/lib/docker-context.js'; import { cwdCache } from '../../../src/lib/hooks/useActorConfig.js'; import { useConsoleSpy } from '../../__setup__/hooks/useConsoleSpy.js'; import { useIsolatedGit } from '../../__setup__/hooks/useIsolatedGit.js'; @@ -15,6 +19,8 @@ import { useTempPath } from '../../__setup__/hooks/useTempPath.js'; const actName = 'push-monorepo-actor'; const ACTOR_ID = 'mOnO1234efGh'; const ACTOR_PATH = 'actors/typescript-actor'; +// As on macOS and Windows by default: this file is found under its name in upper case too. +const caseInsensitiveFileSystem = existsSync(fileURLToPath(import.meta.url).toUpperCase()); const actor = { id: ACTOR_ID, @@ -26,10 +32,19 @@ const actor = { const build = { id: 'build1', actId: ACTOR_ID, buildNumber: '0.0.1', status: 'SUCCEEDED' }; const currentVersion = { versionNumber: '0.0', sourceType: ACTOR_SOURCE_TYPES.SOURCE_FILES }; -const actorGet = vitest.fn(async () => actor); +interface VersionPayload { + sourceType: string; + sourceFiles?: { name: string; content?: string }[]; + tarballUrl?: string; + actorPath?: string; +} + +const actorGet = vitest.fn(async (): Promise => actor); +const actorsCreate = vitest.fn(async (_created: unknown) => actor); const versionUpdate = vitest.fn(async (_modifier: unknown) => ({})); const versionCreate = vitest.fn(async (created: unknown) => created); -const keyValueStoreGetOrCreate = vitest.fn(async () => ({ id: 'store1' })); +const keyValueStoreGetOrCreate = vitest.fn(async () => ({ id: 'store1', urlSigningSecretKey: 'secret' })); +const setRecord = vitest.fn(async (_record: unknown) => {}); // Stands in for the platform: the Actor and its version exist, and every write is recorded. vitest.mock('../../../src/lib/utils.js', async (importOriginal) => ({ @@ -45,16 +60,38 @@ vitest.mock('../../../src/lib/utils.js', async (importOriginal) => ({ versions: () => ({ create: versionCreate }), build: async () => build, }), + actors: () => ({ create: actorsCreate }), keyValueStores: () => ({ getOrCreate: keyValueStoreGetOrCreate }), + keyValueStore: () => ({ setRecord }), build: () => ({ get: async () => build }), })), })); +// Creating an Actor looks its template up in the manifest; nothing here needs the network. +vitest.mock('@apify/actor-templates', () => ({ fetchManifest: async () => ({ templates: [] }) })); + // The mocked `node:process` copy has no signal handlers; nothing here needs them. vitest.mock('../../../src/lib/hooks/useAbortJobOnSignal.js', () => ({ useAbortJobOnSignal: () => ({ [Symbol.dispose]() {} }), })); +// With `gitAvailable` off, every git call fails as it does without git, so the fixture counts as outside Git +// even though it lives inside this repository's checkout. +const gitState = vitest.hoisted(() => ({ gitAvailable: true })); +vitest.mock('node:child_process', async (importOriginal) => { + const original = await importOriginal(); + const unavailable = () => { + throw new Error('not a git repository'); + }; + return { + ...original, + execSync: ((...args: Parameters) => + gitState.gitAvailable ? original.execSync(...args) : unavailable()) as typeof original.execSync, + execFileSync: ((...args: Parameters) => + gitState.gitAvailable ? original.execFileSync(...args) : unavailable()) as typeof original.execFileSync, + }; +}); + const { beforeAllCalls, afterAllCalls, joinPath, tmpPath, forceNewCwd } = useTempPath(actName, { create: true, remove: true, @@ -62,6 +99,9 @@ const { beforeAllCalls, afterAllCalls, joinPath, tmpPath, forceNewCwd } = useTem cwdParent: false, }); +// A sibling folder of the repository: the target of links that leave it. +const outside = useTempPath('push-monorepo-outside', { create: true, remove: true, cwd: false, cwdParent: false }); + // Git reads the global excludes file, so the developer's own one could leave out the files these tests push. useIsolatedGit('push-monorepo-git-home'); @@ -87,6 +127,11 @@ function git(...args: string[]) { execFileSync('git', args, { cwd: tmpPath, stdio: 'ignore' }); } +function versionPayload(mock: typeof versionUpdate | typeof actorsCreate) { + expect(mock).toHaveBeenCalledTimes(1); + return mock.mock.calls[0][0] as VersionPayload; +} + // The command sets its exit code on the mocked copy, but the framework records a thrown error on the real `process`. let originalExitCode: typeof process.exitCode; let originalRealExitCode: typeof process.exitCode; @@ -94,12 +139,16 @@ let originalRealExitCode: typeof process.exitCode; beforeEach(async () => { originalExitCode = commandProcess.exitCode; originalRealExitCode = process.exitCode; - actorGet.mockClear(); - versionUpdate.mockClear(); - versionCreate.mockClear(); + gitState.gitAvailable = true; + actorGet.mockReset(); + actorsCreate.mockReset(); + versionUpdate.mockReset(); + versionCreate.mockReset(); keyValueStoreGetOrCreate.mockClear(); + setRecord.mockClear(); cwdCache.clear(); await beforeAllCalls(); + await outside.beforeAllCalls(); // The layout of apify/actor-monorepo-example, as its own Git repository. await write('package.json', '{"workspaces":["actors/**","packages/**"]}'); @@ -119,43 +168,721 @@ afterEach(async () => { commandProcess.exitCode = originalExitCode; process.exitCode = originalRealExitCode; await afterAllCalls(); + await outside.afterAllCalls(); }); describe('apify push of a monorepo Actor', () => { - it('stops before touching the platform, pointing at the tracking issue', async () => { + it('uploads the repository root as source files with the Actor folder as actorPath', async () => { await testRunCommand(ActorsPushCommand, {}); - expect(commandProcess.exitCode).toBe(CommandExitCodes.NotImplemented); - const stderr = logMessages.error.join('\n'); - expect(stderr).toContain( - `Actor ${actName} sets "dockerContextDir" in .actor/actor.json, so it builds from a Docker context outside its own folder (a monorepo Actor).`, + expect(commandProcess.exitCode).toBeFalsy(); + const modifier = versionPayload(versionUpdate); + expect(modifier.sourceType).toBe(ACTOR_SOURCE_TYPES.SOURCE_FILES); + expect(modifier.actorPath).toBe(ACTOR_PATH); + expect(modifier.tarballUrl).toBeUndefined(); + const names = modifier.sourceFiles!.map((file) => file.name); + expect(names).toEqual( + expect.arrayContaining([ + `${ACTOR_PATH}/.actor/actor.json`, + `${ACTOR_PATH}/src/index.ts`, + 'package.json', + 'shared/TypeScript_Dockerfile', + 'packages/typescript-utils/src/index.ts', + ]), ); - expect(stderr).toContain('apify push does not support such Actors on the Apify platform yet'); - expect(stderr).toContain(MONOREPO_PLATFORM_SUPPORT_ISSUE_URL); - expect(stderr).toContain("set the Actor's source in Apify Console to the Git repository with the folder"); - expect(actorGet).not.toHaveBeenCalled(); - expect(versionCreate).not.toHaveBeenCalled(); - expect(versionUpdate).not.toHaveBeenCalled(); + expect(names.some((name) => name.startsWith('node_modules'))).toBe(false); + expect(names.some((name) => name.includes('\\'))).toBe(false); expect(keyValueStoreGetOrCreate).not.toHaveBeenCalled(); + expect(logMessages.error.join('\n')).not.toContain('uncommitted changes'); + expect(logMessages.log.join('\n')).toContain('Apify push result: SUCCEEDED'); }); - it('keeps stdout clean with --json', async () => { + it('keeps the --json envelope on stdout', async () => { await testRunCommand(ActorsPushCommand, { flags_json: true }); - expect(commandProcess.exitCode).toBe(CommandExitCodes.NotImplemented); - expect(logMessages.log).toEqual([]); - expect(logMessages.error.join('\n')).toContain(MONOREPO_PLATFORM_SUPPORT_ISSUE_URL); + expect(commandProcess.exitCode).toBeFalsy(); + expect(versionPayload(versionUpdate).actorPath).toBe(ACTOR_PATH); + expect(logMessages.log).toHaveLength(1); + expect(JSON.parse(logMessages.log[0])).toMatchObject({ ok: true, operation: 'push', actor: { id: ACTOR_ID } }); + }); + + it('zips a repository over the multi-file limit into the OS temp directory and pushes it as a tarball', async () => { + await write(`${ACTOR_PATH}/big.bin`, 'x'.repeat(MAX_MULTIFILE_BYTES + 1)); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + expect(keyValueStoreGetOrCreate).toHaveBeenCalledWith(`actor-${ACTOR_ID}-source`); + expect(setRecord).toHaveBeenCalledTimes(1); + const record = setRecord.mock.calls[0][0] as { key: string; value: Buffer; contentType: string }; + expect(record.key).toMatch(/^version-0\.0-monorepo-[0-9a-f]{16}\.zip$/); + expect(record.contentType).toBe('application/zip'); + expect(Buffer.isBuffer(record.value)).toBe(true); + expect(record.value.length).toBeGreaterThan(0); + + const modifier = versionPayload(versionUpdate); + expect(modifier.sourceType).toBe(ACTOR_SOURCE_TYPES.TARBALL); + expect(modifier.sourceFiles).toBeUndefined(); + expect(modifier.actorPath).toBe(ACTOR_PATH); + const tarballUrl = new URL(modifier.tarballUrl!); + expect(tarballUrl.pathname).toBe(`/v2/key-value-stores/store1/records/${record.key}`); + expect(tarballUrl.searchParams.get('disableRedirect')).toBe('true'); + expect(tarballUrl.searchParams.get('signature')).toBeTruthy(); + + expect(existsSync(join(process.cwd(), 'temp_file.zip'))).toBe(false); + expect(existsSync(joinPath('temp_file.zip'))).toBe(false); + }); + + it('zips the upload with the repository root at the top level, where the platform resolves actorPath', async () => { + await write(`${ACTOR_PATH}/big.bin`, 'x'.repeat(MAX_MULTIFILE_BYTES + 1)); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const record = setRecord.mock.calls[0][0] as { value: Buffer }; + const names = new AdmZip(record.value).getEntries().map((entry) => entry.entryName); + expect(names).toEqual( + expect.arrayContaining(['package.json', `${versionPayload(versionUpdate).actorPath}/.actor/actor.json`]), + ); + }); + + it('gives each actorPath its own zip record, so a failed update leaves the record of the version as it was', async () => { + await write(`${ACTOR_PATH}/big.bin`, 'x'.repeat(MAX_MULTIFILE_BYTES + 1)); + await testRunCommand(ActorsPushCommand, {}); + expect(commandProcess.exitCode).toBeFalsy(); + + // The version now builds from that record with the old actorPath; a push after a rename then fails its update. + renameSync(joinPath(ACTOR_PATH), joinPath('actors/renamed-actor')); + forceNewCwd('actors/renamed-actor'); + cwdCache.clear(); + versionUpdate.mockRejectedValueOnce(new Error('network down')); + await testRunCommand(ActorsPushCommand, { flags_force: true }); + + expect(logMessages.error.join('\n')).toContain('network down'); + const keys = setRecord.mock.calls.map(([record]) => (record as { key: string }).key); + expect(keys).toHaveLength(2); + expect(keys[1]).not.toBe(keys[0]); + }); + + it('zips a repository under the multi-file limit when a file name exceeds what the platform accepts', async () => { + const longPath = `packages/${'a'.repeat(120)}/${'b'.repeat(120)}/fixture.json`; + await write(longPath, '{}'); + // Over 260 characters, Git on Windows skips the folder unless told otherwise, and the system config of a + // Windows runner, which does, doesn't apply here. + git('config', 'core.longpaths', 'true'); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const modifier = versionPayload(versionUpdate); + expect(modifier.sourceType).toBe(ACTOR_SOURCE_TYPES.TARBALL); + expect(modifier.sourceFiles).toBeUndefined(); + const record = setRecord.mock.calls[0][0] as { value: Buffer }; + expect(new AdmZip(record.value).getEntry(longPath)).toBeTruthy(); + }); + + it('zips a repository under the multi-file limit whose content grows over it as sent, as base64 does to binary files', async () => { + // Raw bytes under the limit, but a binary file goes as base64, which the platform counts at four thirds. + await write(`${ACTOR_PATH}/image.bin`, '\u0000'.repeat(Math.floor(MAX_MULTIFILE_BYTES * 0.9))); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const modifier = versionPayload(versionUpdate); + expect(modifier.sourceType).toBe(ACTOR_SOURCE_TYPES.TARBALL); + expect(modifier.sourceFiles).toBeUndefined(); + expect(setRecord).toHaveBeenCalledTimes(1); + }); + + it('zips a repository whose text holds emoji the platform counts over the multi-file limit', async () => { + // Four bytes each in UTF-8, so the raw size stays under the limit, but the platform counts five. + await write(`${ACTOR_PATH}/emoji.txt`, '\u{1F600}'.repeat(700_000)); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + expect(versionPayload(versionUpdate).sourceType).toBe(ACTOR_SOURCE_TYPES.TARBALL); + }); + + it('leaves no zip in the OS temp directory when getting the source store fails', async () => { + await write(`${ACTOR_PATH}/big.bin`, 'x'.repeat(MAX_MULTIFILE_BYTES + 1)); + const pushZips = () => readdirSync(tmpdir()).filter((name) => /^apify-push-.*\.zip$/.test(name)); + const zipsBefore = pushZips(); + keyValueStoreGetOrCreate.mockRejectedValueOnce(new Error('network down')); + + await testRunCommand(ActorsPushCommand, {}); + + expect(process.exitCode).toBe(1); + expect(logMessages.error.join('\n')).toContain('network down'); + expect(pushZips().filter((name) => !zipsBefore.includes(name))).toEqual([]); + }); + + it('zips a file reached through a symlink as its content, as the multi-file upload sends it', async () => { + await write(`${ACTOR_PATH}/big.bin`, 'x'.repeat(MAX_MULTIFILE_BYTES + 1)); + await write('shared/input_schema.json', '{"title":"Input","type":"object","properties":{}}'); + symlinkSync('../../../shared/input_schema.json', joinPath(`${ACTOR_PATH}/.actor/input_schema.json`)); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const record = setRecord.mock.calls[0][0] as { value: Buffer }; + const entry = new AdmZip(record.value).getEntry(`${ACTOR_PATH}/.actor/input_schema.json`); + // A link entry would hold the link's target path, which the platform unzips as a link to a file it lacks. + expect(entry?.getData().toString()).toContain('"title":"Input"'); + }); + + it('uploads the files under a directory symlink as copies and leaves dangling and outside links out', async () => { + symlinkSync('../../shared', joinPath(`${ACTOR_PATH}/shared`)); + symlinkSync('missing', joinPath('dangling')); + await writeFile(outside.joinPath('outside.txt'), 'outside\n'); + symlinkSync(outside.joinPath('outside.txt'), joinPath('outside.txt')); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const names = versionPayload(versionUpdate).sourceFiles!.map((file) => file.name); + expect(names).toContain(`${ACTOR_PATH}/shared/TypeScript_Dockerfile`); + expect(names).toContain('shared/TypeScript_Dockerfile'); + expect(names).not.toContain(`${ACTOR_PATH}/shared`); + expect(names).not.toContain('dangling'); + expect(names).not.toContain('outside.txt'); + }); + + it('sends a link to a file the upload holds as that file and leaves out one to a file the upload lacks', async () => { + // A workspace package shares the root .env, which only the root .gitignore line covers. + await write('.gitignore', 'node_modules\n/.env\n'); + await write('.env', 'API_TOKEN=secret\n'); + symlinkSync('../../.env', joinPath('packages/typescript-utils/.env')); + symlinkSync('../.git/config', joinPath('shared/git-config')); + symlinkSync('../package.json', joinPath('shared/package.json')); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const files = versionPayload(versionUpdate).sourceFiles!; + const names = files.map((file) => file.name); + expect(names).not.toContain('packages/typescript-utils/.env'); + expect(names).not.toContain('shared/git-config'); + expect(files.find((file) => file.name === 'shared/package.json')?.content).toContain('workspaces'); + }); + + it('sends a link elsewhere in the repository to a file only the Actor folder adds', async () => { + // The Actor folder's .actorignore takes back its gitignored dist, which a clone lacks. + await write('.gitignore', 'node_modules\ndist\n'); + await write(`${ACTOR_PATH}/.actorignore`, '!dist/\n'); + await write(`${ACTOR_PATH}/dist/main.js`, 'console.log(1);\n'); + symlinkSync(`../${ACTOR_PATH}/dist/main.js`, joinPath('shared/main.js')); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const files = versionPayload(versionUpdate).sourceFiles!; + expect(files.find((file) => file.name === 'shared/main.js')?.content).toContain('console.log(1)'); + }); + + it('sends a directory link elsewhere in the repository to files only the Actor folder adds as copies', async () => { + await write('.gitignore', 'node_modules\ndist\n'); + await write(`${ACTOR_PATH}/.actorignore`, '!dist/\n'); + await write(`${ACTOR_PATH}/dist/main.js`, 'console.log(1);\n'); + symlinkSync(`../${ACTOR_PATH}/dist`, joinPath('shared/actor-dist')); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const files = versionPayload(versionUpdate).sourceFiles!; + expect(files.find((file) => file.name === 'shared/actor-dist/main.js')?.content).toContain('console.log(1)'); + }); + + it('copies the files under a directory link inside a linked directory, as a clone resolves both, and ends a cycle', async () => { + await write('config/app.json', '{"app":true}'); + symlinkSync('packages/typescript-utils', joinPath('linked-utils')); + symlinkSync('../../config', joinPath('packages/typescript-utils/config')); + symlinkSync('.', joinPath('packages/typescript-utils/self')); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const files = versionPayload(versionUpdate).sourceFiles!; + const names = files.map((file) => file.name); + expect(names).toEqual( + expect.arrayContaining([ + 'linked-utils/src/index.ts', + 'packages/typescript-utils/config/app.json', + 'packages/typescript-utils/self/src/index.ts', + ]), + ); + expect(files.find((file) => file.name === 'linked-utils/config/app.json')?.content).toBe('{"app":true}'); + expect(names.some((name) => name.includes('/self/self/'))).toBe(false); + }); + + it('accepts a schema reached through a symlink inside the repository and uploads it at the link path', async () => { + await write('shared/input_schema.json', '{"title":"Input","type":"object","properties":{}}'); + symlinkSync('../../../shared/input_schema.json', joinPath(`${ACTOR_PATH}/.actor/input_schema.json`)); + await writeActorConfig({ + dockerContextDir: '../../..', + dockerfile: '../../../shared/TypeScript_Dockerfile', + input: './input_schema.json', + }); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const files = versionPayload(versionUpdate).sourceFiles!; + const linked = files.find((file) => file.name === `${ACTOR_PATH}/.actor/input_schema.json`); + expect(linked?.content).toContain('"title":"Input"'); + }); + + it('accepts a Dockerfile in a Docker context reached through a symlink inside the repository', async () => { + // `context-link` links to the shared folder; the upload carries it as a normal folder under the link's name. + symlinkSync('shared', joinPath('context-link')); + await writeActorConfig({ + dockerContextDir: '../../../context-link', + dockerfile: '../../../context-link/TypeScript_Dockerfile', + }); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const names = versionPayload(versionUpdate).sourceFiles!.map((file) => file.name); + expect(names).toContain('context-link/TypeScript_Dockerfile'); + }); + + it('rejects a schema reached through a symlink leaving the repository, which the upload leaves out', async () => { + await writeFile(outside.joinPath('input_schema.json'), '{}'); + symlinkSync(outside.joinPath('input_schema.json'), joinPath(`${ACTOR_PATH}/.actor/input_schema.json`)); + await writeActorConfig({ + dockerContextDir: '../../..', + dockerfile: '../../../shared/TypeScript_Dockerfile', + input: './input_schema.json', + }); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBe(CommandExitCodes.InvalidActorJson); + const stderr = logMessages.error.join('\n'); + expect(stderr).toContain('"input": "./input_schema.json"'); + expect(stderr).toContain('is not among the files apify push uploads'); + expect(actorGet).not.toHaveBeenCalled(); expect(versionUpdate).not.toHaveBeenCalled(); }); - it('names the Actor folder when actor.json has no name', async () => { - await writeActorConfig({ name: undefined, dockerContextDir: '../../..' }); + it('warns about uncommitted changes, which the upload carries as they are', async () => { + await write(`${ACTOR_PATH}/src/index.ts`, 'console.log(2);\n'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + expect(versionUpdate).toHaveBeenCalledTimes(1); + const warningLine = logMessages.error.find((line) => line.includes('uncommitted changes')); + expect(warningLine).toBeDefined(); + expect(warningLine!.split('\n')).toHaveLength(1); + }); + + it('creates a new Actor with actorPath on its inline version and on the version it pushes', async () => { + actorGet.mockResolvedValueOnce(null); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + expect(versionPayload(actorsCreate)).toMatchObject({ versions: [{ actorPath: ACTOR_PATH }] }); + expect(versionPayload(versionUpdate).actorPath).toBe(ACTOR_PATH); + }); + + it('uploads the Docker context with .actorignore applied outside Git', async () => { + // The fixture's .git stays, though git itself is unavailable: it ends the walk for .gitignore files above the + // fixture, so the .gitignore of this checkout never applies. + await write('.actorignore', 'packages/\n'); + // Local run state of the Actor, which an ordinary push of its folder leaves out too. + await write(`${ACTOR_PATH}/node_modules/dep/index.js`, 'module.exports = 1;\n'); + await write(`${ACTOR_PATH}/storage/key_value_stores/default/INPUT.json`, '{}'); + gitState.gitAvailable = false; + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const modifier = versionPayload(versionUpdate); + expect(modifier.sourceType).toBe(ACTOR_SOURCE_TYPES.SOURCE_FILES); + expect(modifier.actorPath).toBe(ACTOR_PATH); + const names = modifier.sourceFiles!.map((file) => file.name); + expect(names).toEqual( + expect.arrayContaining([`${ACTOR_PATH}/.actor/actor.json`, 'package.json', 'shared/TypeScript_Dockerfile']), + ); + expect(names.some((name) => name.startsWith('packages/'))).toBe(false); + expect(names.some((name) => name.startsWith('node_modules'))).toBe(false); + expect( + names.filter((name) => ['node_modules', 'storage'].some((dir) => name.startsWith(`${ACTOR_PATH}/${dir}/`))), + ).toEqual([]); + }); + + it('leaves out the files under a folder symlink leading out of the Docker context outside Git', async () => { + await mkdir(outside.joinPath('folder'), { recursive: true }); + await writeFile(outside.joinPath('folder/outside.txt'), 'outside\n'); + symlinkSync(outside.joinPath('folder'), joinPath('external')); + symlinkSync('shared', joinPath('internal')); + gitState.gitAvailable = false; - await testRunCommand(ActorsPushCommand, { args_actorId: ACTOR_ID }); + await testRunCommand(ActorsPushCommand, {}); - expect(commandProcess.exitCode).toBe(CommandExitCodes.NotImplemented); - expect(logMessages.error.join('\n')).toContain('Actor typescript-actor sets "dockerContextDir"'); + expect(commandProcess.exitCode).toBeFalsy(); + const names = versionPayload(versionUpdate).sourceFiles!.map((file) => file.name); + expect(names).toContain('internal/TypeScript_Dockerfile'); + expect(names.filter((name) => name.startsWith('external/'))).toEqual([]); + }); + + it('rejects a dockerfile outside the Docker context before touching the platform', async () => { + await writeActorConfig({ dockerContextDir: '../..', dockerfile: '../../../shared/TypeScript_Dockerfile' }); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBe(CommandExitCodes.InvalidActorJson); + expect(logMessages.error.join('\n')).toContain('lies outside the Docker context'); expect(actorGet).not.toHaveBeenCalled(); + expect(versionUpdate).not.toHaveBeenCalled(); + expect(keyValueStoreGetOrCreate).not.toHaveBeenCalled(); + }); + + describe('with the Docker context at the repository root and no dockerfile field', () => { + beforeEach(async () => { + await writeActorConfig({ dockerContextDir: '../../..' }); + }); + + it('pushes with the default Dockerfile in the Actor folder, which lies inside the context', async () => { + await write(`${ACTOR_PATH}/Dockerfile`, 'FROM apify/actor-node:20\n'); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const modifier = versionPayload(versionUpdate); + expect(modifier.actorPath).toBe(ACTOR_PATH); + expect(modifier.sourceFiles!.map((file) => file.name)).toContain(`${ACTOR_PATH}/Dockerfile`); + expect(logMessages.log.join('\n')).toContain('Apify push result: SUCCEEDED'); + }); + + it('exits as an invalid actor.json without a Dockerfile at the default locations', async () => { + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBe(CommandExitCodes.InvalidActorJson); + expect(logMessages.error.join('\n')).toContain('"dockerfile" is not set'); + expect(actorGet).not.toHaveBeenCalled(); + expect(versionUpdate).not.toHaveBeenCalled(); + }); + }); + + it('adds the files of an Actor folder ignored by .gitignore when --dir names it, as a clone has none of them', async () => { + await write('.gitignore', 'node_modules\nactors/\n'); + git('rm', '-r', '--cached', '--quiet', 'actors'); + forceNewCwd('.'); + + await testRunCommand(ActorsPushCommand, { flags_dir: ACTOR_PATH }); + + expect(commandProcess.exitCode).toBeFalsy(); + const modifier = versionPayload(versionUpdate); + expect(modifier.actorPath).toBe(ACTOR_PATH); + expect(modifier.sourceFiles!.map((file) => file.name)).toEqual( + expect.arrayContaining([ + `${ACTOR_PATH}/.actor/actor.json`, + `${ACTOR_PATH}/src/index.ts`, + 'shared/TypeScript_Dockerfile', + ]), + ); + }); + + it("applies the Actor folder's .actorignore, negations included, and leaves the rest of the repository as a clone has it", async () => { + await write('.gitignore', 'node_modules\ndist\n'); + await write(`${ACTOR_PATH}/.actorignore`, 'local-secrets.json\n!dist/\n'); + await write(`${ACTOR_PATH}/local-secrets.json`, '{}'); + await write(`${ACTOR_PATH}/dist/main.js`, 'console.log(1);\n'); + await write('packages/typescript-utils/dist/index.js', 'export const x = 1;\n'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const names = versionPayload(versionUpdate).sourceFiles!.map((file) => file.name); + expect(names).toEqual( + expect.arrayContaining([`${ACTOR_PATH}/dist/main.js`, 'packages/typescript-utils/src/index.ts']), + ); + expect(names).not.toContain(`${ACTOR_PATH}/local-secrets.json`); + expect(names).not.toContain('packages/typescript-utils/dist/index.js'); + }); + + it('pushes an Actor folder whose name is stored decomposed with one copy of its files under actorPath', async () => { + // Finder stores names decomposed (NFD), and git on macOS lists them composed (NFC). + const decomposedPath = 'actors/café-actor'; + await write( + `${decomposedPath}/.actor/actor.json`, + JSON.stringify({ + actorSpecification: 1, + name: actName, + version: '0.0', + buildTag: 'latest', + dockerContextDir: '../../..', + dockerfile: '../../../shared/TypeScript_Dockerfile', + }), + ); + git('add', '.'); + forceNewCwd(decomposedPath); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const { actorPath, sourceFiles } = versionPayload(versionUpdate); + const names = sourceFiles!.map((file) => file.name); + expect(names).toContain(`${actorPath}/.actor/actor.json`); + expect(names.filter((name) => name.normalize('NFC') === 'actors/café-actor/.actor/actor.json')).toHaveLength(1); + }); + + it.runIf(caseInsensitiveFileSystem)( + 'pushes an Actor folder renamed by case only, which Git still lists in its old case, with one copy of its files', + async () => { + await write(`${ACTOR_PATH}/.actorignore`, 'fixtures/\n'); + await write(`${ACTOR_PATH}/fixtures/big.json`, '{}'); + git('add', '.'); + git('-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '--quiet', '-m', 'fixtures'); + // Without git mv, Git on a case-insensitive file system keeps the old case in its index. + renameSync(joinPath(ACTOR_PATH), joinPath('actors/renaming')); + renameSync(joinPath('actors/renaming'), joinPath('actors/TypeScript-Actor')); + forceNewCwd('actors/TypeScript-Actor'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const { actorPath, sourceFiles } = versionPayload(versionUpdate); + const names = sourceFiles!.map((file) => file.name); + expect(actorPath).toBe('actors/TypeScript-Actor'); + expect(names.filter((name) => name.toLowerCase().startsWith('actors/typescript-actor/fixtures/'))).toEqual([]); + expect(names.filter((name) => name.toLowerCase() === 'actors/typescript-actor/.actor/actor.json')).toHaveLength( + 1, + ); + }, + ); + + it.runIf(caseInsensitiveFileSystem)( + 'sends the siblings of the Actor folder under the folder name actorPath spells after a case-only rename', + async () => { + await write('actors/lib/util.js', 'export const y = 2;\n'); + git('add', '.'); + git('-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '--quiet', '-m', 'lib'); + renameSync(joinPath('actors'), joinPath('renaming')); + renameSync(joinPath('renaming'), joinPath('Actors')); + forceNewCwd('Actors/typescript-actor'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const { actorPath, sourceFiles } = versionPayload(versionUpdate); + const names = sourceFiles!.map((file) => file.name); + expect(actorPath).toBe('Actors/typescript-actor'); + expect(names).toEqual(expect.arrayContaining(['Actors/lib/util.js', 'Actors/typescript-actor/src/index.ts'])); + expect(names.filter((name) => name.startsWith('actors/'))).toEqual([]); + }, + ); + + it.runIf(caseInsensitiveFileSystem)( + 'copies through directory links after a case-only rename of the Actor folder, its gitignored dist included, under one spelling', + async () => { + await write('.gitignore', 'node_modules\ndist\n'); + await write(`${ACTOR_PATH}/.actorignore`, '!dist/\n'); + await write(`${ACTOR_PATH}/dist/main.js`, 'console.log(1);\n'); + symlinkSync(`../${ACTOR_PATH}/dist`, joinPath('shared/actor-dist')); + symlinkSync('actors', joinPath('linked-actors')); + git('add', '.'); + git('-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '--quiet', '-m', 'links'); + renameSync(joinPath(ACTOR_PATH), joinPath('actors/renaming')); + renameSync(joinPath('actors/renaming'), joinPath('actors/TypeScript-Actor')); + forceNewCwd('actors/TypeScript-Actor'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const names = versionPayload(versionUpdate).sourceFiles!.map((file) => file.name); + expect(names).toEqual( + expect.arrayContaining(['shared/actor-dist/main.js', 'linked-actors/typescript-actor/src/index.ts']), + ); + expect(names.filter((name) => name.startsWith('linked-actors/TypeScript-Actor/'))).toEqual([]); + }, + ); + + it.runIf(!caseInsensitiveFileSystem)( + 'keeps a link named like a folder of the Actor folder in another case as a folder of its own', + async () => { + await write('actors/lib/util.js', 'export const y = 2;\n'); + symlinkSync('actors', joinPath('Actors')); + git('add', '.'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const names = versionPayload(versionUpdate).sourceFiles!.map((file) => file.name); + expect(names).toEqual(expect.arrayContaining(['actors/lib/util.js', 'Actors/lib/util.js'])); + }, + ); + + it.runIf(!caseInsensitiveFileSystem)( + 'copies a gitignored Actor file through a directory link when a tracked file differs from it in case only', + async () => { + await write('.gitignore', 'node_modules\ndist\n'); + await write(`${ACTOR_PATH}/.actorignore`, '!dist/\n'); + await write(`${ACTOR_PATH}/dist/main.js`, 'console.log(1);\n'); + await write(`${ACTOR_PATH}/Dist/main.js`, 'console.log(2);\n'); + symlinkSync(`../${ACTOR_PATH}/dist`, joinPath('shared/actor-dist')); + git('add', '.'); + git('add', '--force', `${ACTOR_PATH}/Dist/main.js`); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const files = versionPayload(versionUpdate).sourceFiles!; + expect(files.find((file) => file.name === 'shared/actor-dist/main.js')?.content).toContain('console.log(1)'); + }, + ); + + // As on macOS: git lists the folder composed (NFC) while it is stored decomposed (NFD). + it.runIf(process.platform === 'darwin')( + 'sends the siblings of an Actor folder in a folder stored decomposed under the name actorPath spells', + async () => { + const decomposedParent = 'packages/cafe\u0301'; + await write( + `${decomposedParent}/actor/.actor/actor.json`, + JSON.stringify({ + actorSpecification: 1, + name: actName, + version: '0.0', + dockerContextDir: '../../../..', + dockerfile: '../../../../shared/TypeScript_Dockerfile', + }), + ); + await write(`${decomposedParent}/lib/util.py`, 'x = 1\n'); + git('add', '.'); + forceNewCwd(`${decomposedParent}/actor`); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const { actorPath, sourceFiles } = versionPayload(versionUpdate); + const names = sourceFiles!.map((file) => file.name); + expect(actorPath).toBe(`${decomposedParent}/actor`); + expect(names).toEqual( + expect.arrayContaining([`${decomposedParent}/lib/util.py`, `${decomposedParent}/actor/.actor/actor.json`]), + ); + expect(names.filter((name) => name.startsWith('packages/caf\u00e9/'))).toEqual([]); + }, + ); + + // As on macOS: a name composed (NFC) finds the folder stored decomposed (NFD). + it.runIf(process.platform === 'darwin')( + 'uploads links elsewhere that compose the name of an Actor folder stored decomposed', + async () => { + const decomposedPath = 'actors/cafe\u0301-actor'; + const composedPath = 'actors/caf\u00e9-actor'; + await write( + `${decomposedPath}/.actor/actor.json`, + JSON.stringify({ + actorSpecification: 1, + name: actName, + version: '0.0', + dockerContextDir: '../../..', + dockerfile: '../../../shared/TypeScript_Dockerfile', + }), + ); + await write(`${decomposedPath}/src/main.js`, 'console.log(1);\n'); + symlinkSync(`../${composedPath}/src/main.js`, joinPath('shared/main.js')); + symlinkSync(`../${composedPath}/src`, joinPath('shared/actor-src')); + git('add', '.'); + forceNewCwd(decomposedPath); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const names = versionPayload(versionUpdate).sourceFiles!.map((file) => file.name); + expect(names).toEqual(expect.arrayContaining(['shared/main.js', 'shared/actor-src/main.js'])); + }, + ); + + it.runIf(caseInsensitiveFileSystem)( + 'uploads links elsewhere that spell an Actor folder renamed by case only in its old case', + async () => { + symlinkSync(`../${ACTOR_PATH}/src/index.ts`, joinPath('shared/index.ts')); + symlinkSync(`../${ACTOR_PATH}/src`, joinPath('shared/actor-src')); + git('add', '.'); + git('-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '--quiet', '-m', 'links'); + renameSync(joinPath(ACTOR_PATH), joinPath('actors/renaming')); + renameSync(joinPath('actors/renaming'), joinPath('actors/TypeScript-Actor')); + forceNewCwd('actors/TypeScript-Actor'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const names = versionPayload(versionUpdate).sourceFiles!.map((file) => file.name); + expect(names).toEqual(expect.arrayContaining(['shared/index.ts', 'shared/actor-src/index.ts'])); + }, + ); + + it.runIf(caseInsensitiveFileSystem)( + 'uploads a link in the Actor folder whose target names the repository in another letter case', + async () => { + const repositoryInUpperCase = basename(tmpPath).toUpperCase(); + symlinkSync( + `../../../${repositoryInUpperCase}/packages/typescript-utils/src/index.ts`, + joinPath(`${ACTOR_PATH}/case-link.ts`), + ); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const names = versionPayload(versionUpdate).sourceFiles!.map((file) => file.name); + expect(names).toContain(`${ACTOR_PATH}/case-link.ts`); + }, + ); + + describe.runIf(caseInsensitiveFileSystem)('with --dir typed in another case', () => { + it.each([ + [ + 'a monorepo Actor with actorPath in the case on disk', + { dockerContextDir: '../../..', dockerfile: '../../../shared/TypeScript_Dockerfile' }, + ACTOR_PATH, + ], + [ + 'an Actor with the Docker context inside its folder', + { dockerContextDir: '../src', dockerfile: '../src/Dockerfile' }, + undefined, + ], + ])('pushes %s', async (_, fields, expectedActorPath) => { + await write(`${ACTOR_PATH}/src/Dockerfile`, 'FROM apify/actor-node:20\n'); + await writeActorConfig(fields); + forceNewCwd('.'); + + await testRunCommand(ActorsPushCommand, { flags_dir: 'actors/TypeScript-Actor' }); + + expect(commandProcess.exitCode).toBeFalsy(); + const modifier = versionPayload(versionUpdate); + expect(modifier.actorPath).toBe(expectedActorPath); + expect(modifier.sourceFiles!.some((file) => file.name.startsWith('actors/TypeScript-Actor/'))).toBe(false); + }); + }); + + it('checks the actor.json paths from the real Actor folder when --dir goes through a symlink', async () => { + symlinkSync(ACTOR_PATH, joinPath('linked-actor')); + forceNewCwd('linked-actor'); + + await testRunCommand(ActorsPushCommand, {}); + + expect(commandProcess.exitCode).toBeFalsy(); + const modifier = versionPayload(versionUpdate); + expect(modifier.actorPath).toBe(ACTOR_PATH); + expect(modifier.sourceFiles!.map((file) => file.name)).toContain(`${ACTOR_PATH}/.actor/actor.json`); }); it.each([ @@ -189,12 +916,57 @@ describe('apify push of a monorepo Actor', () => { expect(commandProcess.exitCode).toBeFalsy(); expect(process.exitCode).toBeFalsy(); - expect(versionUpdate).toHaveBeenCalledTimes(1); - const modifier = versionUpdate.mock.calls[0][0] as { sourceType: string; sourceFiles: { name: string }[] }; + const modifier = versionPayload(versionUpdate); expect(modifier.sourceType).toBe(ACTOR_SOURCE_TYPES.SOURCE_FILES); - expect(modifier.sourceFiles.map((file) => file.name).sort()).toEqual( + expect(modifier.sourceFiles!.map((file) => file.name).sort()).toEqual( ['.actor/actor.json', 'src/index.ts', ...extraFiles].sort(), ); + expect(modifier).not.toHaveProperty('actorPath'); expect(logMessages.log.join('\n')).toContain('Apify push result: SUCCEEDED'); }); + + it('pushes each Actor with --all from the repository root, a monorepo one as the root with its own actorPath', async () => { + const SECOND_ACTOR_PATH = 'actors/python-actor'; + const PLAIN_ACTOR_PATH = 'tools/plain-actor'; + await write( + `${SECOND_ACTOR_PATH}/.actor/actor.json`, + JSON.stringify({ + actorSpecification: 1, + name: 'push-monorepo-python', + version: '0.0', + buildTag: 'latest', + dockerContextDir: '../../..', + dockerfile: '../../../shared/TypeScript_Dockerfile', + }), + ); + await write(`${SECOND_ACTOR_PATH}/main.py`, 'print(1)\n'); + // Without dockerContextDir, an Actor in the same repository pushes its own folder. + await write( + `${PLAIN_ACTOR_PATH}/.actor/actor.json`, + JSON.stringify({ actorSpecification: 1, name: 'push-plain', version: '0.0', buildTag: 'latest' }), + ); + await write(`${PLAIN_ACTOR_PATH}/main.js`, 'console.log(1);\n'); + git('add', '.'); + git('-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '--quiet', '-m', 'more Actors'); + forceNewCwd('.'); + + await testRunCommand(ActorsPushCommand, { flags_all: true }); + + expect(commandProcess.exitCode).toBeFalsy(); + expect(versionUpdate).toHaveBeenCalledTimes(3); + const [second, first, plain] = versionUpdate.mock.calls.map(([modifier]) => modifier as VersionPayload); + expect(second.actorPath).toBe(SECOND_ACTOR_PATH); + expect(first.actorPath).toBe(ACTOR_PATH); + for (const modifier of [second, first]) { + expect(modifier.sourceFiles!.map((file) => file.name)).toEqual( + expect.arrayContaining([ + `${SECOND_ACTOR_PATH}/.actor/actor.json`, + `${ACTOR_PATH}/.actor/actor.json`, + 'shared/TypeScript_Dockerfile', + ]), + ); + } + expect(plain).not.toHaveProperty('actorPath'); + expect(plain.sourceFiles!.map((file) => file.name).sort()).toEqual(['.actor/actor.json', 'main.js']); + }); }); diff --git a/test/local/lib/docker-context-files.test.ts b/test/local/lib/docker-context-files.test.ts index 0632678e2..5fcb3eef1 100644 --- a/test/local/lib/docker-context-files.test.ts +++ b/test/local/lib/docker-context-files.test.ts @@ -1,6 +1,6 @@ import { execFileSync } from 'node:child_process'; import { mkdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import { dirname, join } from 'node:path'; import { getContextFilePaths } from '../../../src/lib/docker-context.js'; import { useTempPath } from '../../__setup__/hooks/useTempPath.js'; @@ -43,6 +43,8 @@ describe('getContextFilePaths', () => { await beforeAllCalls(); write('package.json', '{}'); + // Git sorts names by their bytes, so this name comes first in its list. + write(' notes.txt'); write('.gitignore', 'dist\n'); write('shared/Dockerfile', 'FROM apify/actor-node:20\n'); write(`${ACTOR_PATH}/.actor/actor.json`, '{}'); @@ -85,6 +87,10 @@ describe('getContextFilePaths', () => { expect(paths).not.toContain(`${ACTOR_PATH}/dist/main.js`); }); + it('keeps the space at the start of a name', async () => { + expect(await getContextFilePaths(tmpPath)).toContain(' notes.txt'); + }); + it('skips a tracked file that is deleted locally', async () => { expect(await getContextFilePaths(tmpPath)).not.toContain('packages/utils/src/index.ts'); }); @@ -102,6 +108,73 @@ describe('getContextFilePaths', () => { }); }); + describe('during an unresolved merge', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath('docker-context-files-merge-dir', { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + const git = (...args: string[]) => + execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', ...args], { + cwd: tmpPath, + stdio: 'ignore', + }); + + beforeAll(async () => { + await beforeAllCalls(); + + git('init', '--quiet'); + writeFileSync(joinPath('shared.txt'), 'base\n'); + git('add', '.'); + git('commit', '--quiet', '-m', 'base'); + git('checkout', '--quiet', '-b', 'other'); + writeFileSync(joinPath('shared.txt'), 'other\n'); + git('commit', '--quiet', '-am', 'other'); + git('checkout', '--quiet', '-'); + writeFileSync(joinPath('shared.txt'), 'main\n'); + git('commit', '--quiet', '-am', 'main'); + // The merge stops on the conflict and leaves shared.txt in the index once per stage. + expect(() => git('merge', '--quiet', 'other')).toThrow(); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('lists a conflicted path once', async () => { + expect(await getContextFilePaths(tmpPath)).toEqual(['shared.txt']); + }); + }); + + describe('with a file list over 1 MiB', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath('docker-context-files-large-dir', { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + writeFileSync(joinPath('package.json'), '{}'); + execFileSync('git', ['init', '--quiet'], { cwd: tmpPath, stdio: 'ignore' }); + // A workspace package's own node_modules, which no .gitignore covers: Git lists over 1 MiB of untracked names. + const dependency = joinPath('packages/utils/node_modules/dependency'); + mkdirSync(dependency, { recursive: true }); + for (let i = 0; i < 5000; i++) writeFileSync(join(dependency, `${i}-${'x'.repeat(200)}.js`), ''); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('still lists the files as Git has them, without the untracked node_modules', async () => { + expect(await getContextFilePaths(tmpPath)).toEqual(['package.json']); + }); + }); + describe('without git', () => { const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath('docker-context-files-no-git-dir', { create: true, diff --git a/test/local/lib/docker-context-realpath.test.ts b/test/local/lib/docker-context-realpath.test.ts new file mode 100644 index 000000000..bab554bbf --- /dev/null +++ b/test/local/lib/docker-context-realpath.test.ts @@ -0,0 +1,69 @@ +import { execFileSync } from 'node:child_process'; +import { mkdirSync, type PathLike, realpathSync, writeFileSync } from 'node:fs'; + +import { checkActorPaths } from '../../../src/lib/actor-paths.js'; +import { resolveDockerContext } from '../../../src/lib/docker-context.js'; +import { useTempPath } from '../../__setup__/hooks/useTempPath.js'; + +// vi.mock is hoisted before imports, so the resolver and the path check get the mocked version. With `nativeFails` on, +// the native realpath fails for every path, as on Windows RAM and virtual disks, while the JS one still works. +const fsState = vi.hoisted(() => ({ nativeFails: false })); +vi.mock('node:fs', async (importOriginal) => { + const original = await importOriginal(); + const native = (path: PathLike) => { + if (fsState.nativeFails) { + throw Object.assign(new Error('EISDIR: illegal operation on a directory, realpath'), { code: 'EISDIR' }); + } + return original.realpathSync.native(path); + }; + return { ...original, realpathSync: Object.assign((path: PathLike) => original.realpathSync(path), { native }) }; +}); + +const ACTOR_PATH = 'actors/x'; + +describe('where the native realpath call fails', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath('docker-context-realpath-test-dir', { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + let root: string; + + beforeAll(async () => { + await beforeAllCalls(); + mkdirSync(joinPath(ACTOR_PATH, '.actor'), { recursive: true }); + writeFileSync(joinPath(ACTOR_PATH, '.actor', 'actor.json'), '{}'); + writeFileSync(joinPath(ACTOR_PATH, 'Dockerfile'), 'FROM apify/actor-node:20\n'); + execFileSync('git', ['init', '--quiet'], { cwd: tmpPath, stdio: 'ignore' }); + root = realpathSync(tmpPath); + fsState.nativeFails = true; + }); + + afterAll(async () => { + fsState.nativeFails = false; + await afterAllCalls(); + }); + + it('resolves a monorepo Actor with the JS implementation', () => { + expect(resolveDockerContext(joinPath(ACTOR_PATH), '../../..')).toEqual({ + kind: 'outside', + contextRoot: root, + sourceRoot: root, + actorPath: ACTOR_PATH, + }); + }); + + it('checks the paths of an ordinary Actor with the JS implementation', () => { + const actorDir = joinPath(ACTOR_PATH); + expect( + checkActorPaths({ + actorDir, + uploadRoot: actorDir, + uploadedFiles: ['.actor/actor.json', 'Dockerfile'], + config: { dockerfile: '../Dockerfile' }, + }), + ).toEqual([]); + }); +}); diff --git a/test/local/lib/docker-context-state.test.ts b/test/local/lib/docker-context-state.test.ts index 322f3ac68..860651d90 100644 --- a/test/local/lib/docker-context-state.test.ts +++ b/test/local/lib/docker-context-state.test.ts @@ -73,3 +73,26 @@ describe('readGitState', () => { expect(readGitState(tmpPath)).toEqual({}); }); }); + +describe('readGitState before the first commit', () => { + const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath('docker-context-state-unborn-dir', { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + beforeAll(async () => { + await beforeAllCalls(); + writeFileSync(joinPath('README.md'), 'content'); + execFileSync('git', ['init', '--quiet'], { cwd: tmpPath, stdio: 'ignore' }); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('flags the files as dirty, as no commit has them yet', () => { + expect(readGitState(tmpPath)).toEqual({ dirty: true }); + }); +}); diff --git a/test/local/lib/docker-context.test.ts b/test/local/lib/docker-context.test.ts index 4e92bbde7..beb3bba43 100644 --- a/test/local/lib/docker-context.test.ts +++ b/test/local/lib/docker-context.test.ts @@ -156,6 +156,24 @@ describe('resolveDockerContext', () => { }); }); + it('rejects an absolute path, which the platform resolves from .actor/', () => { + for (const value of [root, join(root, ACTOR_PATH)]) { + expect(resolveDockerContext(actorDir(), value)).toEqual({ + kind: 'invalid', + message: `"dockerContextDir" in .actor/actor.json is absolute (${value}); the path is relative to .actor/.`, + }); + } + }); + + it('rejects a path with backslashes, which the platform reads as part of a name', () => { + expect(resolveDockerContext(actorDir(), '..\\..\\..')).toEqual({ + kind: 'invalid', + message: + '"dockerContextDir" in .actor/actor.json uses a backslash (..\\..\\..); ' + + 'the platform resolves paths with forward slashes.', + }); + }); + it('rejects a field that is not a string', () => { expect(resolveDockerContext(actorDir(), 42)).toEqual({ kind: 'invalid', @@ -164,6 +182,38 @@ describe('resolveDockerContext', () => { }); }); + describe('with the Actor folder at the Git repository root', () => { + const { joinPath, beforeAllCalls, afterAllCalls } = useTempPath('docker-context-root-actor-dir', { + create: true, + remove: true, + cwd: false, + cwdParent: false, + }); + + let repo: string; + + beforeAll(async () => { + await beforeAllCalls(); + mkdirSync(joinPath('repo', '.actor'), { recursive: true }); + symlinkSync('repo', joinPath('link')); + execFileSync('git', ['init', '--quiet'], { cwd: joinPath('repo'), stdio: 'ignore' }); + repo = realpathSync(joinPath('repo')); + }); + + afterAll(async () => { + await afterAllCalls(); + }); + + it('rejects a context that leaves the Actor folder through a symlink leading back to it', () => { + expect(resolveDockerContext(joinPath('repo'), '../../link')).toEqual({ + kind: 'invalid', + message: + `"dockerContextDir" in .actor/actor.json leaves the Actor folder, then leads back into it at ${repo}, ` + + 'through a symlink or a folder name in another letter case. Write the path without leaving the Actor folder.', + }); + }); + }); + describe('outside a Git repository', () => { const { tmpPath, joinPath, beforeAllCalls, afterAllCalls } = useTempPath('docker-context-no-git-dir', { create: true, @@ -179,6 +229,7 @@ describe('resolveDockerContext', () => { await beforeAllCalls(); mkdirSync(joinPath(ACTOR_PATH, '.actor'), { recursive: true }); mkdirSync(joinPath('other'), { recursive: true }); + symlinkSync('x', joinPath('actors', 'x-link')); root = realpathSync(tmpPath); gitState.gitAvailable = false; }); @@ -210,5 +261,14 @@ describe('resolveDockerContext', () => { message: `"dockerContextDir" in .actor/actor.json points to ${join(root, 'missing')}, which does not exist.`, }); }); + + it('rejects a context that leaves the Actor folder through a symlink leading back to it', () => { + expect(resolveDockerContext(actorDir(), '../../x-link')).toEqual({ + kind: 'invalid', + message: + `"dockerContextDir" in .actor/actor.json leaves the Actor folder, then leads back into it at ${join(root, ACTOR_PATH)}, ` + + 'through a symlink or a folder name in another letter case. Write the path without leaving the Actor folder.', + }); + }); }); });