fix: generate portals, SDKs and plugins from projects with a large static/ folder

src/actions/plugin/generate.ts

@@ -100,9 +100,8 @@ export class PluginGenerateAction {
       const tempContext = new TempContext(tempDirectory);
       const written = await configContext.recordLanguages(selection);
       return await written
-        .asyncAndThen(() => new ResultAsync(configContext.stageUpload(tempDirectory, selection)))
         .mapErr((failure): PluginGenerationProblem => ({ kind: 'configNotPrepared', failure }))
-        .andThen((staged) => new ResultAsync(tempContext.zip(staged)))
+        .asyncAndThen(() => new ResultAsync(project.pluginBuildZip(tempDirectory, selection)))
         .andThen((upload) =>
           new ResultAsync(
             this.prompts.generatePlugin(

src/actions/portal/prepare-project.ts

@@ -80,9 +80,15 @@ export class PreparePortalProjectAction {
     // The artifacts live in this directory for as long as the caller needs them, so it wraps
     // everything that reads them rather than being opened and closed around the call.
     return await withDirPath(async (artifactsDirectory) => {
+      const build = await project.buildZip(artifactsDirectory);
+      if (build.isErr()) {
+        this.prompts.sourceNotZipped(build.error, sourceDirectory);
+        return ActionResult.failed();
+      }
+
       const artifacts = await this.prompts.generateArtifacts(
         this.artifactsService.generate(
-          sourceDirectory,
+          build.value,
           artifactsDirectory,
           this.configDir,
           this.commandMetadata,

src/actions/sdk/generate.ts

@@ -71,15 +71,15 @@ export class GenerateAction {
 
     return await withDirPath(async (tempDirectory) => {
       const tempContext = new TempContext(tempDirectory);
-      const srcDirZip = await buildFrom.srcDirZip(tempDirectory, packageSettingsDirectory);
-      if (srcDirZip.isErr()) {
-        this.prompts.srcDirNotZipped(sourceDirectory, srcDirZip.error);
+      const build = await buildFrom.buildZip(tempDirectory, packageSettingsDirectory);
+      if (build.isErr()) {
+        this.prompts.srcDirNotZipped(sourceDirectory, build.error);
         return ActionResult.failed();
       }
 
       const response = await this.prompts.generateSdk(
         this.sdkGenerationService.generateSdk(
-          srcDirZip.value,
+          build.value,
           language,
           stability,
           this.configDir,

src/infrastructure/file-service.ts

@@ -242,19 +242,41 @@ export class FileService {
     await this.forEachEntry(source, destination, (from, to) => fsExtra.copy(from, to));
   }
 
+  /** Copies what links point to, so relative ones still resolve. `excluding` ignores case, as Windows and macOS do. */
+  public async copyDirectoryContentsFollowingLinks(
+    source: DirectoryPath,
+    destination: DirectoryPath,
+    excluding: readonly FileName[] = []
+  ) {
+    await this.forEachEntry(
+      source,
+      destination,
+      (from, to) => fsExtra.copy(from, to, { dereference: true }),
+      excluding
+    );
+  }
+
   public async moveDirectoryContents(source: DirectoryPath, destination: DirectoryPath) {
     await this.forEachEntry(source, destination, (from, to) => fsExtra.move(from, to, { overwrite: true }));
   }
 
   private async forEachEntry(
     source: DirectoryPath,
     destination: DirectoryPath,
-    operation: (from: string, to: string) => Promise<void>
+    operation: (from: string, to: string) => Promise<void>,
+    excluding: readonly FileName[] = []
   ) {
     const entries = await fsExtra.readdir(source.toString());
-    await Promise.all(
-      entries.map((entry) => operation(path.join(source.toString(), entry), path.join(destination.toString(), entry)))
+    // Settled rather than raced, so no entry is still being written when the caller cleans up after a failure.
+    const outcomes = await Promise.allSettled(
+      entries
+        .filter((entry) => !excluding.some((name) => name.is(entry)))
+        .map((entry) => operation(path.join(source.toString(), entry), path.join(destination.toString(), entry)))
     );
+    const failure = outcomes.find((outcome): outcome is PromiseRejectedResult => outcome.status === 'rejected');
+    if (failure) {
+      throw failure.reason;
+    }
   }
 
   public async deleteFile(filePath: FilePath): Promise<void> {

src/infrastructure/services/portal-artifacts-service.ts

@@ -50,12 +50,11 @@ export class PortalArtifactsService {
    * One run of `/portal-artifacts`: the SDKs, the code-sample catalogs and the context plugin,
    * delivered as a single archive and unpacked into `into`.
    *
-   * `source` is the `src/` directory, which is zipped and uploaded as the build. What the run
-   * produces is decided entirely by the `apimatic.json` inside it — the languages it names, and
-   * whether it carries a `plugin` block.
+   * `build` is the zipped `src/` directory. What the run produces is decided entirely by the
+   * `apimatic.json` inside it — the languages it names, and whether it carries a `plugin` block.
    */
   public async generate(
-    source: DirectoryPath,
+    build: FilePath,
     into: DirectoryPath,
     configDir: DirectoryPath,
     commandMetadata: CommandMetadata,
@@ -69,12 +68,6 @@ export class PortalArtifactsService {
       return err(ServiceError.unauthorizedWithHint(null));
     }
 
-    const build = new FilePath(into, new FileName('build.zip'));
-    const archived = await this.zipService.archive(source, build);
-    if (archived.isErr()) {
-      return err(ServiceError.InvalidResponse);
-    }
-
     const initiated = await this.initiateGeneration(build, commandMetadata, token);
     if (initiated.isErr()) {
       return err(initiated.error);

src/prompts/api/validate.ts

@@ -7,7 +7,7 @@ import { ValidateApiResult, ValidationEntry, ValidationSummary } from '@apimatic
 import { FilePath } from '../../types/file/filePath.js';
 import { ResolveProblem } from '../../types/resource-context.js';
 import { format as f } from '../format.js';
-import { withSpinner } from '../prompt.js';
+import { notZippedForUpload, withSpinner } from '../prompt.js';
 
 export class ApiValidatePrompts {
   public async validateApi<E>(fn: Promise<Result<ValidateApiResult, E>>) {
@@ -106,7 +106,7 @@ export class ApiValidatePrompts {
         return;
       }
       case 'zipFailed': {
-        log.error(`${f.path(problem.specDirectory)} could not be zipped for upload: ${problem.reason}`);
+        log.error(notZippedForUpload(problem.specDirectory, problem.reason));
         return;
       }
       case 'symlinks': {

src/prompts/plugin/generate.ts

@@ -5,7 +5,7 @@ import { PublishingProfileItem } from '../../types/publish-api/publishing-profil
 import { DirectoryPath } from '../../types/file/directoryPath.js';
 import { FilePath } from '../../types/file/filePath.js';
 import { format as f } from '../format.js';
-import { withSpinner } from '../prompt.js';
+import { notZippedForUpload, withSpinner } from '../prompt.js';
 import { APIMATIC_CONFIG_FILE_NAME } from '../../types/apimatic-config/document.js';
 import { PluginConfig } from '../../types/plugin-config-context.js';
 import { PluginGenerationProblem } from '../../types/plugin/generation-problem.js';
@@ -24,7 +24,7 @@ function describeGenerationProblem(problem: PluginGenerationProblem, sourceDirec
     case 'generationFailed':
       return problem.error.errorMessage;
     case 'zipFailed':
-      return `${f.path(sourceDirectory)} could not be zipped for upload: ${problem.reason}`;
+      return notZippedForUpload(sourceDirectory, problem.reason);
     case 'unzipFailed':
       return `${ServiceError.InvalidResponse.errorMessage}\n${problem.reason}`;
   }

src/prompts/portal/prepare-project.ts

@@ -4,13 +4,15 @@ import { ServiceError } from '../../infrastructure/service-error.js';
 import { PortalAuthorizationFailure } from '../../infrastructure/services/portal-authorization-service.js';
 import { DirectoryPath } from '../../types/file/directoryPath.js';
 import { FileName } from '../../types/file/fileName.js';
+import { FileProblem } from '../../types/file/file-problem.js';
 import { FilePath } from '../../types/file/filePath.js';
 import { SharedEndpoint, UnplacedSamples } from '../../types/portal/code-samples.js';
 import { ContentNotices } from '../../types/portal/content-notices.js';
 import { MissingArtifacts } from '../../types/portal/generated-pages.js';
 import { PortalArtifacts } from '../../types/portal/portal-artifacts.js';
 import { PortalSourceProblem } from '../../types/portal/portal-source.js';
 import { format as f } from '../format.js';
+import { notZippedForUpload } from '../prompt.js';
 import { describeMissingArtifacts, generateArtifacts } from './artifacts.js';
 import { reportAuthorizationFailure } from './authorization.js';
 import { reportSharedEndpoints, reportUnplacedSamples } from './code-samples.js';
@@ -39,6 +41,10 @@ export class PreparePortalProjectPrompts {
     reportAuthorizationFailure(failure);
   }
 
+  public sourceNotZipped(problem: FileProblem, sourceDirectory: DirectoryPath) {
+    log.error(notZippedForUpload(sourceDirectory, problem.reason));
+  }
+
   public generateArtifacts(fn: Promise<Result<PortalArtifacts, ServiceError>>) {
     return generateArtifacts(fn);
   }

src/prompts/prompt.ts

@@ -4,6 +4,8 @@ import pc from 'picocolors';
 import { getColumns } from '@clack/core';
 import { log, note, NoteOptions, S_BAR_H, S_CONNECT_LEFT, spinner } from '@clack/prompts';
 import { Result } from 'neverthrow';
+import { DirectoryPath } from '../types/file/directoryPath.js';
+import { format as f } from './format.js';
 
 /** A fixed message, or one built from what the operation returned. */
 type SpinnerMessage<T> = string | ((value: T) => string);
@@ -56,6 +58,9 @@ export const noteWrapped = (message: string, title: string) => {
   }
 };
 
+export const notZippedForUpload = (directory: DirectoryPath, reason: string): string =>
+  `${f.path(directory)} could not be zipped for upload: ${reason}`;
+
 type ColumnStyle = 'primary' | 'secondary' | 'item';
 
 export function buildTableWithHeading(

src/prompts/sdk/generate.ts

@@ -3,7 +3,7 @@ import { DirectoryPath } from '../../types/file/directoryPath.js';
 import { FileProblem } from '../../types/file/file-problem.js';
 import { format as f } from '../format.js';
 import { Result } from 'neverthrow';
-import { withSpinner } from '../prompt.js';
+import { notZippedForUpload, withSpinner } from '../prompt.js';
 import { ServiceError } from '../../infrastructure/service-error.js';
 import { AVAILABLE_LANGUAGES, Language, languageLabel, UPCOMING_LANGUAGES } from '../../types/sdk/generate.js';
 import { VersionProblem } from '../../types/project-context.js';
@@ -68,7 +68,7 @@ export class SdkGeneratePrompts {
   }
 
   public srcDirNotZipped(sourceDirectory: DirectoryPath, problem: FileProblem) {
-    log.error(`${f.path(sourceDirectory)} could not be zipped for upload: ${problem.reason}`);
+    log.error(notZippedForUpload(sourceDirectory, problem.reason));
   }
 
   public sdkNotUnzipped(problem: FileProblem) {

src/types/plugin-config-context.ts

@@ -1,5 +1,4 @@
 import { ok, Result, ResultAsync } from 'neverthrow';
-import { FileService } from '../infrastructure/file-service.js';
 import { ApimaticConfigContext, ApimaticConfigWriteFailure } from './apimatic-config-context.js';
 import { ApimaticConfigDocument, ConfigBlockName, findingClause } from './apimatic-config/document.js';
 import { isPublished, languagesOf, PluginLanguageEntry, PluginLanguages } from './apimatic-config/languages-block.js';
@@ -13,7 +12,6 @@ import {
   PluginMetadata
 } from './plugin/plugin-config.js';
 import { PortalLanguages } from './portal/portal-languages.js';
-import { SOURCE_DIRECTORY_NAME } from './project-layout.js';
 import { SemVersion } from './publish/version.js';
 import { isAvailableLanguage, Language } from './sdk/generate.js';
 
@@ -147,7 +145,6 @@ const withMetadata = (
 
 export class PluginConfigContext {
   private readonly configContext: ApimaticConfigContext;
-  private readonly fileService = new FileService();
 
   constructor(private readonly sourceDirectory: DirectoryPath) {
     this.configContext = new ApimaticConfigContext(this.sourceDirectory);
@@ -206,25 +203,15 @@ export class PluginConfigContext {
     return await this.merge((document) => document.with('languages', recorded(document, languages, keepsRecord)));
   }
 
-  /**
-   * The `src/` to upload: a copy whose `languages` names exactly what the plugin covers. The
-   * user's file keeps the published entries this run leaves out; the service reads the copy.
-   */
-  public async stageUpload(
-    into: DirectoryPath,
-    languages: readonly Language[]
-  ): Promise<Result<DirectoryPath, PluginConfigWriteFailure>> {
-    const staged = into.join(SOURCE_DIRECTORY_NAME);
-    await this.fileService.copyDirectoryContents(this.sourceDirectory, staged);
-
-    const config = new ApimaticConfigContext(staged);
-    const covered = await config.merge(OWNED_BLOCKS, (document) =>
+  /** Only for the build's copy: `languages` comes to name just what the plugin covers, dropping published entries. */
+  public async coverOnly(languages: readonly Language[]): Promise<Result<void, PluginConfigWriteFailure>> {
+    const covered = await this.configContext.merge(OWNED_BLOCKS, (document) =>
       document.with('languages', recorded(document, languages, isUnavailableLanguage))
     );
 
     // The merge writes nothing when it changes nothing, so a mark on the copy can outlive it —
     // and the service reads this file with a parser that will not look past one.
-    return await covered.asyncAndThen(() => new ResultAsync(config.removeByteOrderMark())).map(() => staged);
+    return await covered.asyncAndThen(() => new ResultAsync(this.configContext.removeByteOrderMark()));
   }
 
   private async merge(

src/types/plugin/generation-problem.ts

@@ -2,7 +2,6 @@ import { ServiceError } from '../../infrastructure/service-error.js';
 import { FileProblem } from '../file/file-problem.js';
 import { PluginConfigWriteFailure } from '../plugin-config-context.js';
 
-export type PluginGenerationProblem =
-  | FileProblem
-  | { kind: 'configNotPrepared'; failure: PluginConfigWriteFailure }
-  | { kind: 'generationFailed'; error: ServiceError };
+export type PluginBuildProblem = FileProblem | { kind: 'configNotPrepared'; failure: PluginConfigWriteFailure };
+
+export type PluginGenerationProblem = PluginBuildProblem | { kind: 'generationFailed'; error: ServiceError };

src/types/project-context.ts

@@ -1,14 +1,21 @@
 import { err, ok, Result } from 'neverthrow';
 import { FileService } from '../infrastructure/file-service.js';
+import { errorMessage } from '../utils/error-utils.js';
 import { ApimaticConfigContext } from './apimatic-config-context.js';
 import { BuildConfig } from './build/build.js';
 import { DirectoryPath } from './file/directoryPath.js';
 import { FileName } from './file/fileName.js';
 import { FilePath } from './file/filePath.js';
 import { FileProblem } from './file/file-problem.js';
 import { PluginConfigContext } from './plugin-config-context.js';
+import { PluginBuildProblem } from './plugin/generation-problem.js';
 import { PortalSourceContext } from './portal-source-context.js';
-import { OUTPUT_DIRECTORY_NAMES, SOURCE_DIRECTORY_NAME, SPEC_DIRECTORY_NAME } from './project-layout.js';
+import {
+  LOCAL_ONLY_DIRECTORY_NAMES,
+  OUTPUT_DIRECTORY_NAMES,
+  SOURCE_DIRECTORY_NAME,
+  SPEC_DIRECTORY_NAME
+} from './project-layout.js';
 import { Language } from './sdk/generate.js';
 import { SdkContext } from './sdk-context.js';
 import { SpecContext } from './spec-context.js';
@@ -113,16 +120,30 @@ export class ProjectContext {
     );
   }
 
-  public async srcDirZip(
+  public async buildZip(
     tempDirectory: DirectoryPath,
     packageSettingsDirectory?: DirectoryPath
   ): Promise<Result<FilePath, FileProblem>> {
-    const staged = tempDirectory.join(SOURCE_DIRECTORY_NAME);
-    await this.fileService.copyDirectoryContents(this.source, staged);
-    if (packageSettingsDirectory) {
-      await this.fileService.copyDirectoryContents(packageSettingsDirectory, staged.join('package-settings'));
+    const staged = await this.stagedBuild(tempDirectory, packageSettingsDirectory);
+    if (staged.isErr()) {
+      return err(staged.error);
     }
-    return await new TempContext(tempDirectory).zip(staged);
+    return await new TempContext(tempDirectory).zip(staged.value);
+  }
+
+  public async pluginBuildZip(
+    tempDirectory: DirectoryPath,
+    languages: readonly Language[]
+  ): Promise<Result<FilePath, PluginBuildProblem>> {
+    const staged = await this.stagedBuild(tempDirectory);
+    if (staged.isErr()) {
+      return err(staged.error);
+    }
+    const covered = await new PluginConfigContext(staged.value).coverOnly(languages);
+    if (covered.isErr()) {
+      return err({ kind: 'configNotPrepared', failure: covered.error });
+    }
+    return await new TempContext(tempDirectory).zip(staged.value);
   }
 
   public async isVersioned(): Promise<boolean> {
@@ -200,6 +221,25 @@ export class ProjectContext {
     return ok(undefined);
   }
 
+  // A copy that fails is reported as a zip that fails: either way, the build could not be packed.
+  private async stagedBuild(
+    tempDirectory: DirectoryPath,
+    packageSettingsDirectory?: DirectoryPath
+  ): Promise<Result<DirectoryPath, FileProblem>> {
+    const staged = tempDirectory.join(SOURCE_DIRECTORY_NAME);
+    const localOnly = LOCAL_ONLY_DIRECTORY_NAMES.map((name) => new FileName(name));
+    try {
+      await this.fileService.copyDirectoryContentsFollowingLinks(this.source, staged, localOnly);
+      if (packageSettingsDirectory) {
+        const packageSettings = staged.join('package-settings');
+        await this.fileService.copyDirectoryContentsFollowingLinks(packageSettingsDirectory, packageSettings);
+      }
+      return ok(staged);
+    } catch (error) {
+      return err({ kind: 'zipFailed', reason: errorMessage(error) });
+    }
+  }
+
   private reading(versionDirectory: DirectoryPath): ProjectContext {
     return new ProjectContext(this.projectDirectory, versionDirectory, versionDirectory.leafName());
   }

src/types/project-layout.ts

@@ -12,6 +12,9 @@ export const CONTENT_DIRECTORY_NAME = 'content';
 
 export const STATIC_DIRECTORY_NAME = 'static';
 
+/** The folders of `src/` the local site build copies to the site's root as they are, so no build upload sends them. */
+export const LOCAL_ONLY_DIRECTORY_NAMES = [STATIC_DIRECTORY_NAME] as const;
+
 export const OUTPUT_DIRECTORY_NAMES = {
   sdk: 'sdk',
   portal: 'portal',

test/actions/plugin/generate.test.ts

@@ -178,6 +178,17 @@ describe('PluginGenerateAction', () => {
       expect(uploaded['spec/openapi.json']).to.equal('{"openapi": "3.0.0"}');
     });
 
+    it('keeps content/ and leaves out static/', async () => {
+      generated();
+      await fsExtra.outputFile(path.join(sourceDirectory, 'content', 'index.md'), '# Home');
+      await fsExtra.outputFile(path.join(sourceDirectory, 'static', 'images', 'logo.png'), 'png');
+
+      await execute();
+
+      expect(uploaded['content/index.md']).to.equal('# Home');
+      expect(Object.keys(uploaded).filter((entry) => entry.startsWith('static/'))).to.be.empty;
+    });
+
     it('synthesizes nothing: no plugin-config.json is written for the server', async () => {
       generated();