From 8818c660c0104d158c40b6363f9cb10935fd1a81 Mon Sep 17 00:00:00 2001 From: Simon Corry Date: Fri, 4 Sep 2026 16:53:06 -0400 Subject: [PATCH 1/5] Test candidate assessment skill contract --- tests/candidate-assessment-skill.test.js | 128 +++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/candidate-assessment-skill.test.js diff --git a/tests/candidate-assessment-skill.test.js b/tests/candidate-assessment-skill.test.js new file mode 100644 index 0000000..cd14877 --- /dev/null +++ b/tests/candidate-assessment-skill.test.js @@ -0,0 +1,128 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..'); +const skillRoot = join(repoRoot, 'skills', 'candidate-assessment'); + +function read(relativePath) { + return readFileSync(join(skillRoot, relativePath), 'utf8'); +} + +test('candidate assessment is a complete installable skill', () => { + const skill = read('SKILL.md'); + const metadata = read('agents/openai.yaml'); + + assert.match(skill, /^---\nname: candidate-assessment\ndescription: .+\n---\n/); + assert.match(metadata, /display_name: "Candidate Assessment"/); + assert.match(metadata, /default_prompt: "Use \$candidate-assessment/); + + for (const file of [ + 'scoring-and-output.md', + 'hiring-manager.md', + 'portfolio-review.md', + 'design-exercise.md', + 'final-synthesis.md', + 'research-basis.md', + 'ashby-setup.md', + ]) { + assert.match(skill, new RegExp(`\\(references/${file.replace('.', '\\.')}\\)`)); + assert.ok(read(`references/${file}`).trim().length > 0, `${file} should not be empty`); + } +}); + +test('the Ashby output contract keeps its five paste-ready fields and 1 to 4 decision scale', () => { + const scoring = read('references/scoring-and-output.md'); + + for (const heading of [ + '## Overall Recommendation', + '## Reason for Recommendation', + '## Next Steps', + '## Areas to Probe', + '## What Candidate Cares About', + '## Criterion Scores', + ]) { + assert.ok(scoring.includes(heading), `${heading} should remain in the output contract`); + } + + for (const anchor of [ + '### 1: Strong No', + '### 2: No', + '### 3: Yes', + '### 4: Strong Yes', + ]) { + assert.ok(scoring.includes(anchor), `${anchor} should remain defined`); + } + + assert.match(scoring, /Do not average the criterion scores/); + assert.match(scoring, /AI must be at least 3/); +}); + +test('each interview plan preserves its intended evidence and gates', () => { + const hiringManager = read('references/hiring-manager.md'); + const portfolio = read('references/portfolio-review.md'); + const exercise = read('references/design-exercise.md'); + const synthesis = read('references/final-synthesis.md'); + + for (const criterion of [ + 'Distinctive Edge and Generative Curiosity', + 'Problem Finding, Autonomy, and Impact', + 'Velocity, Learning, and Product Judgment', + 'Constructive Challenge, Self-Awareness, and Trust', + 'AI-Native Builder Practice', + ]) { + assert.ok(hiringManager.includes(`### ${criterion}`), `hiring manager should score ${criterion}`); + } + assert.equal((hiringManager.match(/Core gate\./g) ?? []).length, 3); + + for (const criterion of [ + 'Problem Choice and Framing', + 'Personal Ownership and Judgment', + 'Craft, Taste, and Human Intent', + 'Shipping, Customer Contact, and Learning', + 'Distinctive Spike', + 'AI-Enabled Build Practice', + ]) { + assert.ok(portfolio.includes(`### ${criterion}`), `portfolio should score ${criterion}`); + } + assert.equal((portfolio.match(/Core gate\./g) ?? []).length, 3); + + for (const criterion of [ + 'Question Quality and Discovery', + 'Problem Framing and Perspective', + 'Exploration and Distinctive Thinking', + 'Prioritization, Tradeoffs, and Velocity', + 'Collaboration and Adaptation', + 'Direction, Craft, and Human Intent', + ]) { + assert.ok(exercise.includes(`### ${criterion}`), `design exercise should score ${criterion}`); + } + assert.equal((exercise.match(/Core gate\./g) ?? []).length, 2); + assert.match(exercise, /Strong visual output cannot rescue/); + + for (const requirement of [ + 'Problem finding and independent ownership', + 'Product craft, taste, and human judgment', + 'Velocity tied to shipping', + 'AI-native build practice at 3 or higher', + ]) { + assert.ok(synthesis.includes(requirement), `final synthesis should require ${requirement}`); + } +}); + +test('the Ashby setup replaces the inherited generic trait scorecard', () => { + const setup = read('references/ashby-setup.md'); + + for (const plan of [ + '## Hiring Manager Scorecard', + '## Portfolio Review Scorecard', + '## Design Exercise Scorecard', + ]) { + assert.ok(setup.includes(plan), `${plan} should remain configured`); + } + + assert.doesNotMatch(setup, /Trait - Smile|Trait - Fast-brained|Trait - Care and Intensity/); + assert.match(setup, /leave it unscored and say so in the comment/); +}); From 76cb3fb9b9f75b43e70774b7d2372e5d30e288d4 Mon Sep 17 00:00:00 2001 From: Simon Corry Date: Fri, 4 Sep 2026 16:55:18 -0400 Subject: [PATCH 2/5] Keep Ashby criteria aligned with rubric --- tests/candidate-assessment-skill.test.js | 66 +++++++++++++++--------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/tests/candidate-assessment-skill.test.js b/tests/candidate-assessment-skill.test.js index cd14877..f23c2f3 100644 --- a/tests/candidate-assessment-skill.test.js +++ b/tests/candidate-assessment-skill.test.js @@ -7,6 +7,32 @@ import { fileURLToPath } from 'node:url'; const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..'); const skillRoot = join(repoRoot, 'skills', 'candidate-assessment'); +const hiringManagerCriteria = [ + 'Distinctive Edge and Generative Curiosity', + 'Problem Finding, Autonomy, and Impact', + 'Velocity, Learning, and Product Judgment', + 'Constructive Challenge, Self-Awareness, and Trust', + 'AI-Native Builder Practice', +]; + +const portfolioCriteria = [ + 'Problem Choice and Framing', + 'Personal Ownership and Judgment', + 'Craft, Taste, and Human Intent', + 'Shipping, Customer Contact, and Learning', + 'Distinctive Spike', + 'AI-Enabled Build Practice', +]; + +const designExerciseCriteria = [ + 'Question Quality and Discovery', + 'Problem Framing and Perspective', + 'Exploration and Distinctive Thinking', + 'Prioritization, Tradeoffs, and Velocity', + 'Collaboration and Adaptation', + 'Direction, Craft, and Human Intent', +]; + function read(relativePath) { return readFileSync(join(skillRoot, relativePath), 'utf8'); } @@ -33,7 +59,7 @@ test('candidate assessment is a complete installable skill', () => { } }); -test('the Ashby output contract keeps its five paste-ready fields and 1 to 4 decision scale', () => { +test('the Ashby output contract keeps its paste-ready fields and 1 to 4 decision scale', () => { const scoring = read('references/scoring-and-output.md'); for (const heading of [ @@ -66,37 +92,17 @@ test('each interview plan preserves its intended evidence and gates', () => { const exercise = read('references/design-exercise.md'); const synthesis = read('references/final-synthesis.md'); - for (const criterion of [ - 'Distinctive Edge and Generative Curiosity', - 'Problem Finding, Autonomy, and Impact', - 'Velocity, Learning, and Product Judgment', - 'Constructive Challenge, Self-Awareness, and Trust', - 'AI-Native Builder Practice', - ]) { + for (const criterion of hiringManagerCriteria) { assert.ok(hiringManager.includes(`### ${criterion}`), `hiring manager should score ${criterion}`); } assert.equal((hiringManager.match(/Core gate\./g) ?? []).length, 3); - for (const criterion of [ - 'Problem Choice and Framing', - 'Personal Ownership and Judgment', - 'Craft, Taste, and Human Intent', - 'Shipping, Customer Contact, and Learning', - 'Distinctive Spike', - 'AI-Enabled Build Practice', - ]) { + for (const criterion of portfolioCriteria) { assert.ok(portfolio.includes(`### ${criterion}`), `portfolio should score ${criterion}`); } assert.equal((portfolio.match(/Core gate\./g) ?? []).length, 3); - for (const criterion of [ - 'Question Quality and Discovery', - 'Problem Framing and Perspective', - 'Exploration and Distinctive Thinking', - 'Prioritization, Tradeoffs, and Velocity', - 'Collaboration and Adaptation', - 'Direction, Craft, and Human Intent', - ]) { + for (const criterion of designExerciseCriteria) { assert.ok(exercise.includes(`### ${criterion}`), `design exercise should score ${criterion}`); } assert.equal((exercise.match(/Core gate\./g) ?? []).length, 2); @@ -123,6 +129,18 @@ test('the Ashby setup replaces the inherited generic trait scorecard', () => { assert.ok(setup.includes(plan), `${plan} should remain configured`); } + for (const criterion of [ + ...hiringManagerCriteria, + ...portfolioCriteria, + ...designExerciseCriteria, + ]) { + assert.ok(setup.includes(`\`${criterion}\``), `Ashby setup should include ${criterion}`); + } + + for (const label of ['1 Strong No', '2 No', '3 Yes', '4 Strong Yes']) { + assert.ok(setup.includes(label), `Ashby setup should define ${label}`); + } + assert.doesNotMatch(setup, /Trait - Smile|Trait - Fast-brained|Trait - Care and Intensity/); assert.match(setup, /leave it unscored and say so in the comment/); }); From 29dd5fb73bca5f1d1e3adcc686e1e6f3bc9e1ab5 Mon Sep 17 00:00:00 2001 From: Simon Corry Date: Fri, 4 Sep 2026 16:56:04 -0400 Subject: [PATCH 3/5] Scope scorecard contract checks --- tests/candidate-assessment-skill.test.js | 54 +++++++++++++++--------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/tests/candidate-assessment-skill.test.js b/tests/candidate-assessment-skill.test.js index f23c2f3..2d7c61a 100644 --- a/tests/candidate-assessment-skill.test.js +++ b/tests/candidate-assessment-skill.test.js @@ -37,6 +37,22 @@ function read(relativePath) { return readFileSync(join(skillRoot, relativePath), 'utf8'); } +function section(source, heading, nextHeading) { + const start = source.indexOf(heading); + assert.notEqual(start, -1, `${heading} should exist`); + const end = nextHeading ? source.indexOf(nextHeading, start + heading.length) : source.length; + assert.notEqual(end, -1, `${nextHeading} should follow ${heading}`); + return source.slice(start, end); +} + +function levelThreeHeadings(source) { + return [...source.matchAll(/^### (.+)$/gm)].map((match) => match[1]); +} + +function namedScoreFields(source) { + return [...source.matchAll(/^- `([^`]+)`:/gm)].map((match) => match[1]); +} + test('candidate assessment is a complete installable skill', () => { const skill = read('SKILL.md'); const metadata = read('agents/openai.yaml'); @@ -92,19 +108,13 @@ test('each interview plan preserves its intended evidence and gates', () => { const exercise = read('references/design-exercise.md'); const synthesis = read('references/final-synthesis.md'); - for (const criterion of hiringManagerCriteria) { - assert.ok(hiringManager.includes(`### ${criterion}`), `hiring manager should score ${criterion}`); - } + assert.deepEqual(levelThreeHeadings(hiringManager), hiringManagerCriteria); assert.equal((hiringManager.match(/Core gate\./g) ?? []).length, 3); - for (const criterion of portfolioCriteria) { - assert.ok(portfolio.includes(`### ${criterion}`), `portfolio should score ${criterion}`); - } + assert.deepEqual(levelThreeHeadings(portfolio), portfolioCriteria); assert.equal((portfolio.match(/Core gate\./g) ?? []).length, 3); - for (const criterion of designExerciseCriteria) { - assert.ok(exercise.includes(`### ${criterion}`), `design exercise should score ${criterion}`); - } + assert.deepEqual(levelThreeHeadings(exercise), designExerciseCriteria); assert.equal((exercise.match(/Core gate\./g) ?? []).length, 2); assert.match(exercise, /Strong visual output cannot rescue/); @@ -129,16 +139,22 @@ test('the Ashby setup replaces the inherited generic trait scorecard', () => { assert.ok(setup.includes(plan), `${plan} should remain configured`); } - for (const criterion of [ - ...hiringManagerCriteria, - ...portfolioCriteria, - ...designExerciseCriteria, - ]) { - assert.ok(setup.includes(`\`${criterion}\``), `Ashby setup should include ${criterion}`); - } - - for (const label of ['1 Strong No', '2 No', '3 Yes', '4 Strong Yes']) { - assert.ok(setup.includes(label), `Ashby setup should define ${label}`); + assert.deepEqual( + namedScoreFields(section(setup, '## Hiring Manager Scorecard', '## Portfolio Review Scorecard')), + hiringManagerCriteria, + ); + assert.deepEqual( + namedScoreFields(section(setup, '## Portfolio Review Scorecard', '## Design Exercise Scorecard')), + portfolioCriteria, + ); + assert.deepEqual( + namedScoreFields(section(setup, '## Design Exercise Scorecard', '## Field-Level Scale Description')), + designExerciseCriteria, + ); + + const overall = section(setup, '### Overall Recommendation', '### Reason for Recommendation'); + for (const mapping of ['- `4`: Strong Yes', '- `3`: Yes', '- `2`: No', '- `1`: Strong No']) { + assert.ok(overall.includes(mapping), `Overall Recommendation should define ${mapping}`); } assert.doesNotMatch(setup, /Trait - Smile|Trait - Fast-brained|Trait - Care and Intensity/); From 2499a62dba9e61c21336bcce35c5b1f6a0bee12e Mon Sep 17 00:00:00 2001 From: Simon Corry Date: Fri, 4 Sep 2026 16:57:43 -0400 Subject: [PATCH 4/5] Separate startup and AI evidence --- .../candidate-assessment/references/research-basis.md | 7 ++++++- tests/candidate-assessment-skill.test.js | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/skills/candidate-assessment/references/research-basis.md b/skills/candidate-assessment/references/research-basis.md index 49bb05d..db7f6ba 100644 --- a/skills/candidate-assessment/references/research-basis.md +++ b/skills/candidate-assessment/references/research-basis.md @@ -10,15 +10,20 @@ This file explains the durable choices behind the rubric. It is not candidate ev - [Due Diligence interview](https://www.youtube.com/watch?v=Iq6FK3y7x64): look for people who see problems and create without direction. Velocity matters because it creates customer learning, and owning the damage from moving too fast matters just as much. - [The Cost of Context](https://simoncorry.com/blog/2026/08/06/the-cost-of-context), [Design Process, Meet Agent Process](https://simoncorry.com/blog/2026/07/15/design-process-meet-agent-process), and [A Researcher Out of Time](https://simoncorry.com/blog/2026/05/20/a-researcher-out-of-time): strong AI practice is an inspectable, long-lived process with explicit claims, tests, reversals, and human review. -## External Evidence and Startup Practice +## Early-Stage Hiring Evidence - [First Round on the founding designer role](https://review.firstround.com/the-ultimate-guide-to-the-founding-designer-role/): early designers need autonomy, ambiguity tolerance, business prioritization, self-critique, and a lightweight process. Copying a large-company design process is a poor fit. - [Emotive on hiring startup product designers](https://medium.com/emotive-design/how-we-hire-product-designers-at-emotive-12854dfc2b99): early-stage work rewards versatility, context switching, problem-solving, autonomy, and agility. - [Monzo's product design hiring process](https://monzo.com/blog/2019/01/17/monzo-product-designer-jobs): consistent questions, realistic challenges, and attention to framing, collaboration, and tradeoffs produce more useful evidence than puzzle exercises. - [Hartwell et al. on structured interviews](https://www.sciencedirect.com/science/article/pii/S0148296319301985): job-related behavioral and situational questions improve the usefulness of interview evidence. - [Ashby score field guidance](https://www.ashbyhq.com/product-updates/form-sections-question-descriptions-and-scores): explicit criteria, descriptions, comments, and anchored scores support a usable scorecard. + +## AI Practice Evidence + - [Figma's AI Model Designer role](https://job-boards.greenhouse.io/figma/jobs/5711913004) and [Anthropic's product design workflow](https://www-cdn.anthropic.com/58284b19e702b49db9302d5b6f135ad8871e7658.pdf): current AI-native design includes model evaluation, technical understanding, direct implementation, functional prototypes, and attention to states and edge cases. +Figma and Anthropic are evidence for the AI-practice bar only. They are not the model for startup hiring or process. + ## Deliberate Departures From Conventional Rubrics - `Smile` is excluded. Affect is not job evidence. diff --git a/tests/candidate-assessment-skill.test.js b/tests/candidate-assessment-skill.test.js index 2d7c61a..7131dba 100644 --- a/tests/candidate-assessment-skill.test.js +++ b/tests/candidate-assessment-skill.test.js @@ -160,3 +160,13 @@ test('the Ashby setup replaces the inherited generic trait scorecard', () => { assert.doesNotMatch(setup, /Trait - Smile|Trait - Fast-brained|Trait - Care and Intensity/); assert.match(setup, /leave it unscored and say so in the comment/); }); + +test('large-company AI examples do not stand in for startup hiring evidence', () => { + const research = read('references/research-basis.md'); + const startupEvidence = section(research, '## Early-Stage Hiring Evidence', '## AI Practice Evidence'); + const aiEvidence = section(research, '## AI Practice Evidence', '## Deliberate Departures From Conventional Rubrics'); + + assert.doesNotMatch(startupEvidence, /Figma|Anthropic/); + assert.match(aiEvidence, /Figma and Anthropic are evidence for the AI-practice bar only/); + assert.match(aiEvidence, /They are not the model for startup hiring or process/); +}); From 7b3a43f2e1d8520130b34752ad04120058795fbb Mon Sep 17 00:00:00 2001 From: Simon Corry Date: Fri, 4 Sep 2026 17:02:12 -0400 Subject: [PATCH 5/5] Record standalone skill verification --- docs/sessions/LOG.md | 6 +++--- docs/sessions/history/2026-W32.md | 7 +++++++ docs/wiki/INDEX.md | 1 + docs/wiki/engineering/standalone-skills.md | 11 +++++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 docs/sessions/history/2026-W32.md create mode 100644 docs/wiki/engineering/standalone-skills.md diff --git a/docs/sessions/LOG.md b/docs/sessions/LOG.md index 1213047..dca8e99 100644 --- a/docs/sessions/LOG.md +++ b/docs/sessions/LOG.md @@ -2,8 +2,8 @@ One entry per session, newest first, in plain English. Wrap-up writes an entry here at every close and start-up reads this file at every open; that pair is what makes sessions continuous even when a handoff note goes missing. Entries older than the current week move to a history folder beside this file, one file per week, so this file never outgrows a sitting. Entry shape: `## YYYY-MM-DD: Title`, one short paragraph, an optional `Friction:` line when something fought back. -## 2026-08-05: Honest chain economics +## 2026-09-04: Candidate assessment skill -Foundry's cost guidance now follows the evidence instead of treating review rounds as the obvious expense. The full path keeps five rounds; the light path is described honestly as lower assurance. Always-loaded rules and source commands now have enforced size ceilings, the middle review rounds only reread fresh material, live voice checks stay on prose-heavy stages, and test-it plus wrap-up each own one complete check. The implementation challenge chain found six real defects before two clean rounds closed it, including loose policy tests, dead vendor links, and a symlink route around the new budget. +Foundry now carries a standalone Product Design candidate assessment skill for Ashby. It has separate rubrics for hiring manager interviews, portfolio reviews, design exercises, and final decisions, plus five tests that keep the skill, scorecards, and 1 to 4 scale aligned. The follow-up review also separated startup hiring evidence from large-company examples of AI practice. -Friction: All five required implementation rounds found defects, so the chain followed its own confidence rule through round eight; rounds seven and eight were clean. +Friction: The skill reached main before the plan and implementation challenges ran. The missing stages were run afterwards, found three defects in test coverage and research framing, and moved every fix onto one review branch. diff --git a/docs/sessions/history/2026-W32.md b/docs/sessions/history/2026-W32.md new file mode 100644 index 0000000..ba47450 --- /dev/null +++ b/docs/sessions/history/2026-W32.md @@ -0,0 +1,7 @@ +# Sessions, 2026-W32 + +## 2026-08-05: Honest chain economics + +Foundry's cost guidance now follows the evidence instead of treating review rounds as the obvious expense. The full path keeps five rounds; the light path is described honestly as lower assurance. Always-loaded rules and source commands now have enforced size ceilings, the middle review rounds only reread fresh material, live voice checks stay on prose-heavy stages, and test-it plus wrap-up each own one complete check. The implementation challenge chain found six real defects before two clean rounds closed it, including loose policy tests, dead vendor links, and a symlink route around the new budget. + +Friction: All five required implementation rounds found defects, so the chain followed its own confidence rule through round eight; rounds seven and eight were clean. diff --git a/docs/wiki/INDEX.md b/docs/wiki/INDEX.md index 9ce399f..22c7f22 100644 --- a/docs/wiki/INDEX.md +++ b/docs/wiki/INDEX.md @@ -15,6 +15,7 @@ Ideas about building software that have aged well across fifty years, for design - [Context engineering](engineering/context-engineering.md): keeping what the agent reads lean, honest, and re-tunable. Reach for it when a rules file grows, the model generation changes, or the process feels heavier than the work it governs. - [Chain economics](engineering/chain-economics.md): what Foundry's source project actually measured about cost, what stayed uncertain, and which defaults the evidence supports. Reach for it before claiming a process change saves money. - [Clear technical writing](engineering/clear-technical-writing.md): how to explain technical work in plain English without deleting the terms the reader needs. Reach for it when writing for someone who doesn't share your context, or when a review flags unexplained language. +- [Standalone skills](engineering/standalone-skills.md): how to package and verify an optional skill without turning it into another Foundry stage. Reach for it when adding a reusable tool outside the build chain. ## Design fundamentals diff --git a/docs/wiki/engineering/standalone-skills.md b/docs/wiki/engineering/standalone-skills.md new file mode 100644 index 0000000..eff5901 --- /dev/null +++ b/docs/wiki/engineering/standalone-skills.md @@ -0,0 +1,11 @@ +# Standalone skills + +Foundry's named build stages live in the generated command system. A useful skill that is not part of that sequence belongs in its own bundle under `skills/`. This keeps it installable without quietly turning it into another stage for every Foundry project. + +## What belongs in the bundle + +Keep `skills//SKILL.md` at the root of the bundle. Put user-facing metadata in `skills//agents/openai.yaml` and larger instructions in `skills//references/` so the agent reads only what the task needs. The main skill must link every required reference with a relative path. + +## What proves it works + +Folder existence is weak proof. Install the skill from its GitHub path into a temporary directory and compare that result with the source. Add a focused test for the promises that cross files, especially names or scores repeated in setup instructions and evaluation rubrics. Foundry's normal link and jargon checks still apply.