Emit fatal errors as JSON

.changeset/json-fatal-errors.md

@@ -0,0 +1,6 @@
+---
+'@shopify/cli-kit': minor
+'@shopify/cli': minor
+---
+
+Emit machine-readable fatal errors when JSON output is active

docs/cli/error_handling.md

@@ -138,6 +138,35 @@ The `FatalError` pattern will not work well in an architecture where app develop
 
 However, until then, we get a lot of leverage in the CLI from `FatalError`, so it can continue to exist as a high leverage counter-example to some of our general principles.
 
+## Fatal errors in JSON output
+
+When `--json` or `-j` is active, a fatal error writes one document to stdout:
+
+```json
+{"error":{"type":"abort","message":"Couldn't find an app","tryMessage":"Run shopify app config link","nextSteps":["Check that you're in the app directory"]}}
+```
+
+`type` is one of `abort`, `bug`, or `external`. `type` and `message` are always included. The regular error output's optional `tryMessage`, `nextSteps`, and `customSections` content is included as unstyled strings. Link URLs remain visible. Bug errors include `stack`; external errors include `command` and `args`. Other error properties are excluded.
+
+Callers can explicitly attach selected, JSON-serializable data to `error.details`. The renderer includes this field as
+data, so consumers do not need to parse display strings. Do not attach a raw error, request, credentials, or other private
+properties. For example, `store execute` exposes GraphQL errors at `error.details.errors`, including their error codes.
+
+```ts
+const error = new AbortError('GraphQL operation failed.', JSON.stringify({errors}, null, 2))
+error.details = {errors}
+throw error
+```
+
+The handler flushes stdout before returning to Oclif's exit path. If serialization fails, it emits a minimal JSON bug
+error instead of a text banner. Stdout write failures are not retried as serialization failures.
+
+Absent optional fields are omitted. `nextSteps` is an array of strings. Each custom section has an optional `title` and a `body` containing either a string or a string matrix for tabular content.
+
+The process exit code remains the source of truth for success or failure. An `AbortSilentError` remains silent. Recoverable diagnostics and progress are written to stderr so stdout contains only the command result or fatal error document.
+
+This contract applies to execution-level failures. A failure represented by a command's result type belongs in that command's JSON result schema instead.
+
 ## Report a result from a function
 There are scenarios where a function needs to inform the caller about the success or failure of the operation. For that, `@shopify/cli-kit` provides a result utility:
 

packages/cli-kit/src/private/node/json-error.integration.test.ts

@@ -0,0 +1,46 @@
+import {execa} from 'execa'
+import {describe, expect, test} from 'vitest'
+import {fileURLToPath} from 'node:url'
+
+const fixturePath = fileURLToPath(new URL('../../../test/fixtures/json-error-process.ts', import.meta.url))
+
+describe('JSON fatal error process output', () => {
+  test('flushes a large JSON document before Oclif exits', {timeout: 20000}, async () => {
+    const exitFixturePath = fileURLToPath(new URL('../../../test/fixtures/json-error-exit-process.ts', import.meta.url))
+    const result = await execa(process.execPath, ['--loader', 'ts-node/esm', exitFixturePath, '--json'], {
+      env: {SHOPIFY_UNIT_TEST: 'false', FORCE_COLOR: '0', NODE_NO_WARNINGS: '1'},
+      reject: false,
+    })
+
+    expect(result.exitCode, result.stderr).toBe(1)
+    expect(result.stderr).toBe('')
+    expect(JSON.parse(result.stdout)).toStrictEqual({
+      error: {type: 'abort', message: 'x'.repeat(1024 * 1024)},
+    })
+  })
+
+  // Starting a fresh TypeScript subprocess needs extra startup headroom under loaded CI runners.
+  test(
+    'writes one JSON document to stdout, diagnostics to stderr, and preserves the exit code',
+    {timeout: 20000},
+    async () => {
+      const result = await execa(process.execPath, ['--loader', 'ts-node/esm', fixturePath, '--json'], {
+        env: {SHOPIFY_UNIT_TEST: 'false', FORCE_COLOR: '0', NODE_NO_WARNINGS: '1'},
+        reject: false,
+      })
+
+      expect(result.exitCode, result.stderr).toBe(2)
+      expect(JSON.parse(result.stdout)).toStrictEqual({
+        error: {
+          type: 'abort',
+          message: 'Expected failure',
+          tryMessage: 'Run shopify app dev again.',
+          nextSteps: ['Read the documentation (https://shopify.dev).'],
+          customSections: [{title: 'Details', body: 'The app could not be loaded.'}],
+        },
+      })
+      expect(result.stdout.trim().split('\n')).toHaveLength(1)
+      expect(result.stderr).toBe('Recoverable diagnostic')
+    },
+  )
+})

packages/cli-kit/src/private/node/json-error.test.ts

@@ -0,0 +1,260 @@
+import {renderFatalErrorAsJson} from './json-error.js'
+import {AbortError, AbortSilentError, BugError, ExternalError, FatalErrorType} from '../../public/node/error.js'
+import {mockAndCaptureOutput} from '../../public/node/testing/output.js'
+import * as output from '../../public/node/output.js'
+import {afterEach, describe, expect, test, vi} from 'vitest'
+
+afterEach(() => {
+  mockAndCaptureOutput().clear()
+})
+
+function renderedDocument(error: Parameters<typeof renderFatalErrorAsJson>[0]): unknown {
+  const output = mockAndCaptureOutput()
+  output.clear()
+  renderFatalErrorAsJson(error)
+  return JSON.parse(output.info())
+}
+
+describe('renderFatalErrorAsJson', () => {
+  test('includes only explicitly selected structured details', () => {
+    const details = {errors: [{message: 'Invalid field', extensions: {code: 'UNDEFINED_FIELD'}}]}
+    const error = Object.assign(new AbortError('GraphQL operation failed.'), {
+      details,
+      request: {authorization: 'secret'},
+      accessToken: 'secret',
+    })
+
+    expect(renderedDocument(error)).toStrictEqual({
+      error: {type: 'abort', message: 'GraphQL operation failed.', details},
+    })
+  })
+
+  test('falls back to a minimal JSON document when serialization fails', () => {
+    const error = Object.defineProperty(new AbortError('Expected failure'), 'formattedMessage', {
+      get: () => {
+        throw new Error('Could not read formatted message')
+      },
+    })
+
+    expect(renderedDocument(error)).toStrictEqual({
+      error: {type: 'bug', message: 'Failed to serialize the error as JSON.'},
+    })
+  })
+
+  test.each([
+    ['BigInt', () => ({value: BigInt(1)})],
+    [
+      'circular reference',
+      () => {
+        const details: Record<string, unknown> = {}
+        details.self = details
+        return details
+      },
+    ],
+  ] as const)('falls back to JSON when details contain a %s', (_name, createDetails) => {
+    const error = new AbortError('Expected failure')
+    error.details = createDetails()
+
+    expect(renderedDocument(error)).toStrictEqual({
+      error: {type: 'bug', message: 'Failed to serialize the error as JSON.'},
+    })
+  })
+
+  test('does not retry stdout write failures as serialization failures', () => {
+    const error = Object.assign(new Error('write EPIPE'), {code: 'EPIPE'})
+    const write = vi.spyOn(output, 'outputResult').mockImplementation(() => {
+      throw error
+    })
+
+    try {
+      expect(() => renderFatalErrorAsJson(new AbortError('Expected failure'))).toThrow(error)
+      expect(write).toHaveBeenCalledOnce()
+    } finally {
+      write.mockRestore()
+    }
+  })
+
+  test('does not inspect details on silent errors', () => {
+    const error = Object.defineProperty(new AbortSilentError(), 'details', {
+      get: () => {
+        throw new Error('Details must not be read')
+      },
+    })
+
+    renderFatalErrorAsJson(error)
+
+    expect(mockAndCaptureOutput().output()).toBe('')
+  })
+
+  test.each([
+    ['abort', new AbortError('Expected failure'), {type: 'abort', message: 'Expected failure'}],
+    [
+      'bug',
+      new BugError('Unexpected failure'),
+      {type: 'bug', message: 'Unexpected failure', stack: expect.any(String)},
+    ],
+    [
+      'external',
+      new ExternalError('External failure', 'npm', ['install']),
+      {type: 'external', message: 'External failure', command: 'npm', args: ['install']},
+    ],
+  ])('renders a stable %s error', (_type, error, expectedError) => {
+    expect(renderedDocument(error)).toStrictEqual({error: expectedError})
+  })
+
+  test('uses the rich message content and preserves link URLs', () => {
+    const error = new AbortError([
+      'Read',
+      {link: {label: 'the documentation', url: 'https://shopify.dev'}},
+      {char: '.'},
+    ])
+
+    expect(renderedDocument(error)).toStrictEqual({
+      error: {type: 'abort', message: 'Read the documentation (https://shopify.dev).'},
+    })
+  })
+
+  test('includes a plain try message', () => {
+    expect(renderedDocument(new AbortError('Expected failure', 'Try again'))).toStrictEqual({
+      error: {type: 'abort', message: 'Expected failure', tryMessage: 'Try again'},
+    })
+  })
+
+  test('flattens rich try message tokens to an unstyled string', () => {
+    const error = new AbortError('Expected failure', [
+      '\u001B[31mRun\u001B[39m',
+      {command: 'shopify app dev'},
+      {char: '.'},
+    ])
+
+    expect(renderedDocument(error)).toStrictEqual({
+      error: {type: 'abort', message: 'Expected failure', tryMessage: 'Run shopify app dev.'},
+    })
+  })
+
+  test('includes next steps as unstyled strings with visible link URLs', () => {
+    const error = new AbortError('Expected failure', null, [
+      ['Read', {link: {label: 'the documentation', url: 'https://shopify.dev'}}, {char: '.'}],
+      '\u001B[31mTry again.\u001B[39m',
+    ])
+
+    expect(renderedDocument(error)).toStrictEqual({
+      error: {
+        type: 'abort',
+        message: 'Expected failure',
+        nextSteps: ['Read the documentation (https://shopify.dev).', 'Try again.'],
+      },
+    })
+  })
+
+  test('includes custom text and tabular sections', () => {
+    const error = new AbortError('Expected failure', null, undefined, [
+      {
+        title: '\u001B[31mExtension\u001B[39m',
+        body: [
+          {
+            list: {
+              title: 'Validation errors',
+              items: ['Missing name', ['Read', {link: {label: 'the documentation', url: 'https://shopify.dev'}}]],
+            },
+          },
+        ],
+      },
+      {
+        body: {
+          tabularData: [
+            ['Name', {bold: 'Status'}],
+            ['checkout', '\u001B[31mFailed\u001B[39m'],
+          ],
+        },
+      },
+    ])
+
+    expect(renderedDocument(error)).toStrictEqual({
+      error: {
+        type: 'abort',
+        message: 'Expected failure',
+        customSections: [
+          {
+            title: 'Extension',
+            body: 'Validation errors: Missing name; Read the documentation (https://shopify.dev)',
+          },
+          {
+            body: [
+              ['Name', 'Status'],
+              ['checkout', 'Failed'],
+            ],
+          },
+        ],
+      },
+    })
+  })
+
+  test('includes stacks only for bug errors', () => {
+    const bug = new BugError('Unexpected failure')
+    bug.stack = '\u001B[31mError: Unexpected failure\u001B[39m\n    at example.ts:1:1'
+    const abort = new AbortError('Expected failure')
+    abort.stack = 'Error: Expected failure\n    at example.ts:1:1'
+
+    expect(renderedDocument(bug)).toStrictEqual({
+      error: {
+        type: 'bug',
+        message: 'Unexpected failure',
+        stack: 'Error: Unexpected failure\n    at example.ts:1:1',
+      },
+    })
+    expect(renderedDocument(abort)).toStrictEqual({
+      error: {type: 'abort', message: 'Expected failure'},
+    })
+  })
+
+  test('omits empty optional collections', () => {
+    expect(renderedDocument(new AbortError('Expected failure', null, [], []))).toStrictEqual({
+      error: {type: 'abort', message: 'Expected failure'},
+    })
+  })
+
+  test('omits malformed try messages without breaking the base error document', () => {
+    const error = Object.assign(new AbortError('Expected failure'), {tryMessage: {invalid: true}})
+
+    expect(renderedDocument(error)).toStrictEqual({
+      error: {type: 'abort', message: 'Expected failure'},
+    })
+  })
+
+  test('does not render intentionally silent errors', () => {
+    const output = mockAndCaptureOutput()
+    output.clear()
+
+    renderFatalErrorAsJson(new AbortSilentError())
+
+    expect(output.output()).toBe('')
+  })
+
+  test('includes external command context but not external stacks or arbitrary properties', () => {
+    const error = Object.assign(new ExternalError('Safe message', 'npm', ['install']), {
+      stack: 'external stack',
+      accessToken: 'secret',
+      request: {authorization: 'secret'},
+    })
+
+    expect(renderedDocument(error)).toStrictEqual({
+      error: {type: 'external', message: 'Safe message', command: 'npm', args: ['install']},
+    })
+  })
+
+  test('treats unknown fatal error types as bugs', () => {
+    expect(renderedDocument({type: 999, message: 'Future error'})).toStrictEqual({
+      error: {type: 'bug', message: 'Future error'},
+    })
+  })
+
+  test('recognizes silent errors created by another cli-kit copy', () => {
+    const output = mockAndCaptureOutput()
+    output.clear()
+
+    renderFatalErrorAsJson({type: FatalErrorType.AbortSilent, message: ''})
+
+    expect(output.output()).toBe('')
+  })
+})