fix(design): explain unchecked source diagnostics

Author: Bill Leoutsakos

scripts/check-design-conformance-appearance.test.ts

@@ -1,4 +1,4 @@
-/* biome-ignore-all lint/suspicious/noTemplateCurlyInString: Fixtures contain literal proposed JavaScript templates. */
+/** biome-ignore-all lint/suspicious/noTemplateCurlyInString: Fixtures contain literal proposed JavaScript templates. */
 import { existsSync } from 'node:fs'
 import { beforeAll, expect, test } from 'vitest'
 import { extract } from '#design-conformance/extract'

scripts/check-design-conformance-command.test.ts

@@ -104,6 +104,43 @@ test('CI tolerates genuine findings and preserves the report, but fails missing
   expect(run(['--repo', repo, '--base', noCentral, '--head', noCentral], ci).status).toBe(2)
 })
 
+test.each([
+  { label: 'parser failure', source: 'const A=()=> <p className={', reason: 'Parser failure' },
+  {
+    label: 'source limit',
+    source: ' '.repeat(2 * 1024 * 1024 + 1),
+    reason: 'Source exceeds the 2 MiB parsing limit',
+  },
+])('skipped changed files remain actionable without findings: $label', ({ source, reason }) => {
+  const { repo, head: base } = fixture()
+  writeFileSync(path.join(repo, ui), source)
+  commit(repo)
+  const args = ['--repo', repo, '--base', base]
+  const output = path.join(repo, 'report.json')
+  const human = run([...args, '--output', output])
+  expect(human.status).toBe(0)
+  expect(human.stdout).toContain('coverage incomplete')
+  expect(human.stdout).toContain(`${ui}:1 (after`)
+  expect(human.stdout).toContain(reason)
+  const report: Report = JSON.parse(readFileSync(output, 'utf8'))
+  expect(report.status).toBe('completed')
+  expect(report.flagged).toBe(false)
+  expect(report.findings).toEqual([])
+  expect(report.unchecked).toEqual(
+    expect.arrayContaining([
+      expect.objectContaining({ file: ui, side: 'after', reason: expect.stringContaining(reason) }),
+    ])
+  )
+  const summary = path.join(repo, 'summary.md')
+  const child = run(args, ci, { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: summary })
+  expect(child.status).toBe(0)
+  expect(child.stdout).toContain(`${ui}:1 (after`)
+  expect(child.stderr).not.toContain('::warning')
+  const markdown = readFileSync(summary, 'utf8')
+  expect(markdown).toContain(`${ui}:1 (after`)
+  expect(markdown).toContain(reason)
+})
+
 test.each([
   [0, null, { status: 'completed', flagged: false, findings: [] }, 0],
   [1, null, { status: 'completed', flagged: true, findings: [{}] }, 0],
@@ -194,6 +231,49 @@ test('GitHub annotations escape source values and normal logs cannot inject work
   expect(githubSummary(report)).toContain('| Usage violations | 1 |')
 })
 
+test('unchecked diagnostics preserve both sides and safely render source-authored text', () => {
+  const report = new ConformanceLinter().report(null)
+  for (const side of ['before', 'after'])
+    report.unchecked.push({
+      file: 'path`|</pre>\n::error::injected.tsx',
+      line: 7,
+      side,
+      context: 'className',
+      reason: 'Unknown helper: <script>& value\r\n::warning::injected',
+    })
+  const original = JSON.stringify(report)
+  const text = textReport(report)
+  const markdown = githubSummary(report)
+  expect(text).toContain(':7 (before; className)')
+  expect(text).toContain(':7 (after; className)')
+  expect(text).not.toMatch(/[\r\n]::(?:error|warning)::/)
+  expect(markdown).toContain('&lt;/pre&gt;')
+  expect(markdown).toContain('&lt;script&gt;&amp; value')
+  expect(markdown).not.toContain('<script>')
+  expect(markdown).not.toMatch(/[\r\n]::(?:error|warning)::/)
+  expect(githubAnnotations(report)).toEqual([])
+  expect(JSON.stringify(report)).toBe(original)
+})
+
+test('large unchecked summaries show explicit limits while logs retain every complete diagnostic', () => {
+  const report = new ConformanceLinter().report(null)
+  report.unchecked = Array.from({ length: 101 }, (_, index) => ({
+    file: `component-${index}.tsx`,
+    line: 1,
+    side: 'after',
+    context: '',
+    reason: index === 0 ? `${'&'.repeat(2000)} complete-long-diagnostic` : `reason-${index}`,
+  }))
+  const markdown = githubSummary(report)
+  const text = textReport(report)
+  expect(markdown).toContain('Showing 100 of 101 diagnostics')
+  expect(markdown).toContain('[truncated; see check log]')
+  expect(markdown).not.toContain('complete-long-diagnostic')
+  expect(markdown).not.toContain('component-100.tsx')
+  expect(text).toContain('complete-long-diagnostic')
+  expect(text).toContain('component-100.tsx:1 (after) — reason-100')
+})
+
 test('system edits remain flagged but are reported as design review warnings', () => {
   const report = findingReport()
   report.findings[0].kind = 'system-change'

scripts/check-design-conformance-composition.test.ts

@@ -1,4 +1,4 @@
-/* biome-ignore-all lint/suspicious/noTemplateCurlyInString: Proposed source fixtures. */
+/** biome-ignore-all lint/suspicious/noTemplateCurlyInString: Proposed source fixtures. */
 import { expect, test } from 'vitest'
 import { ConformanceLinter } from '#design-conformance/conformance'
 import { extract } from '#design-conformance/extract'

scripts/check-design-conformance-refactors.test.ts

@@ -1,4 +1,4 @@
-/* biome-ignore-all lint/suspicious/noTemplateCurlyInString: Proposed source fixtures. */
+/** biome-ignore-all lint/suspicious/noTemplateCurlyInString: Proposed source fixtures. */
 import { expect, test } from 'vitest'
 import { ConformanceLinter } from '#design-conformance/conformance'
 import { type Change, type Entry, hash, TOKEN_FILE } from '#design-conformance/model'

scripts/check-design-conformance.test.ts

@@ -1,4 +1,4 @@
-/* biome-ignore-all lint/suspicious/noTemplateCurlyInString: Fixtures contain proposed source text. */
+/** biome-ignore-all lint/suspicious/noTemplateCurlyInString: Fixtures contain proposed source text. */
 import { expect, test } from 'vitest'
 import { ConformanceLinter } from '#design-conformance/conformance'
 import { designSystem } from '#design-conformance/design-system'

scripts/design-conformance/README.md

@@ -1,6 +1,6 @@
 # Design conformance check
 
-Tool **3.3.0**, policy **design-conformance/1.3.0**. This is the maintained implementation of Sim's design guardrails.
+Tool **3.3.1**, policy **design-conformance/1.3.0**. This is the maintained implementation of Sim's design guardrails.
 
 From the repository root, using Bun **1.4.1** and the normal root installation:
 
@@ -16,7 +16,7 @@ Use the actual PR target for `--base`; it is required. The repository defaults t
 
 - **Usage violations** identify newly introduced inputs that violate an explicit central design contract, the authoritative definition, and the permitted token/component mechanism. Fix confirmed violations through that mechanism.
 - **System changes** identify edits to central definitions or this registry. Review them as changes to the design system; central authoring may introduce new styling.
-- **Unchecked inputs** describe unresolved syntax or coverage limits. They do not create findings, and a clean result does not prove exhaustive conformance.
+- **Unchecked inputs** describe unresolved syntax or coverage limits. They do not create findings, and a clean result does not prove exhaustive conformance. Text output identifies incomplete coverage and lists every diagnostic with its file, line, comparison side, context and reason. The CI summary includes an expandable list of the first 100 diagnostics, with long entries shortened; complete details remain in the check log and JSON report.
 
 The ordinary command exits **0** for no findings, **1** for findings and **2** for operational failure. JSON retains `flagged: true` even during the warning-only rollout; failed reports have `flagged: null`.
 

scripts/design-conformance/model.ts

@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'
 import { readdirSync, readFileSync } from 'node:fs'
 import type { Route, SourceSummary } from '#design-conformance/source-summary'
 
-export const VERSION = '3.3.0'
+export const VERSION = '3.3.1'
 export const CATALOGUE_VERSION = '1.0.0'
 export type Policy = 'appearance' | 'tokens' | 'conformance'
 export const policyVersion = (policy: Policy) =>

scripts/design-conformance/reporting.ts

@@ -25,12 +25,18 @@ function description(finding: Finding): string {
   return parts.join('; ')
 }
 
+function uncheckedDescription(note: Report['unchecked'][number]): string {
+  return line(
+    `${note.file}:${note.line} (${note.side}${note.context ? `; ${note.context}` : ''}) — ${note.reason}`
+  )
+}
+
 export function textReport(report: Report): string {
   if (report.status === 'failed')
     return `Design check failed: ${line(report.error ?? 'Operational failure')}\n`
   const counts = findingCounts(report)
   const lines = [
-    `Design check: ${report.flagged ? 'findings reported' : 'no new findings'} (${report.policyVersion}).`,
+    `Design check: ${report.flagged ? 'findings reported' : 'no new findings'}${report.unchecked.length ? '; coverage incomplete' : ''} (${report.policyVersion}).`,
     `Usage violations: ${counts.usage}; system changes: ${counts.system}; unchecked diagnostics: ${report.unchecked.length}.`,
   ]
   for (const [kind, title] of [
@@ -47,10 +53,13 @@ export function textReport(report: Report): string {
       )
     }
   }
-  if (report.unchecked.length)
+  if (report.unchecked.length) {
+    lines.push('', 'Unchecked inputs — coverage incomplete')
+    for (const note of report.unchecked) lines.push(`  ${uncheckedDescription(note)}`)
     lines.push(
       'Unchecked inputs remain outside the result; no findings does not prove complete coverage.'
     )
+  }
   return `${lines.join('\n')}\n`
 }
 
@@ -73,7 +82,7 @@ export function githubAnnotations(report: Report): string[] {
 
 export function githubSummary(report: Report): string {
   const counts = findingCounts(report)
-  return [
+  const lines = [
     '### Design conformance',
     '',
     report.status === 'failed'
@@ -91,5 +100,28 @@ export function githubSummary(report: Report): string {
     '',
     'Full findings and authoritative sources are in the check log. Unchecked inputs do not establish conformance.',
     '',
-  ].join('\n')
+  ]
+  if (report.unchecked.length) {
+    /** Bound summary size; the check log retains every complete diagnostic. */
+    const notes = report.unchecked.slice(0, 100)
+    lines.push(
+      '<details>',
+      `<summary>Unchecked inputs (${report.unchecked.length}) — coverage incomplete</summary>`,
+      '',
+      `Showing ${notes.length} of ${report.unchecked.length} diagnostics. Long entries are truncated here. Full details are in the check log; these are not design violations.`,
+      '',
+      '<pre>',
+      ...notes.map((note) => {
+        const text = uncheckedDescription(note)
+        const excerpt =
+          text.length > 1000 ? `${text.slice(0, 1000)}… [truncated; see check log]` : text
+        /** Source text must not close the HTML block or introduce Markdown formatting. */
+        return excerpt.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;')
+      }),
+      '</pre>',
+      '</details>',
+      ''
+    )
+  }
+  return lines.join('\n')
 }