intake: accept new Markdown files next to a tracked file without a component manifest (#364)

lab/internal/molecule/readiness_run.go

@@ -58,6 +58,18 @@ func refuseProjectClaudeConfig(command []string, root, base string) error {
 	return fmt.Errorf("coding refused: base commit %s carries Claude project configuration (%s) that a --setting-sources project run loads, and it can widen permissions or run hooks the Lab did not review; no coding harness was started. Remove it from the repository (for example git rm -r .claude/settings*.json .claude/hooks .claude/agents), commit that, and refresh intake; the Formula has no setting that allows project settings", base, listPaths(found))
 }
 
+// capturedMarkdownFiles returns the captured changes that the plan accepted by
+// the Markdown rule, in capture order.
+func capturedMarkdownFiles(p readiness.Plan, capture verification.CaptureResult) []string {
+	out := []string{}
+	for _, change := range capture.Changes {
+		if slices.Contains(p.MarkdownNewFiles, change) {
+			out = append(out, change)
+		}
+	}
+	return out
+}
+
 // listPaths names at most five paths and counts the rest.
 func listPaths(paths []string) string { return boundedPaths(paths, 5) }
 
@@ -627,6 +639,9 @@ func runIntake(ctx context.Context, o Options, key string) (r Record, err error)
 		p := readiness.Plan{SchemaVersion: "v1", MoleculeID: r.ID, Revision: 1, Inputs: readiness.Inputs{Repository: w.Repo, TargetBranch: target, BaseCommit: w.BaseCommit, FormulaSHA256: f.SHA256, EvidenceSHA256: evidenceHash, BindingSHA256: bindingDigest, Issue: issueInput(issue), MandatoryConstraints: o.MandatoryConstraints}, Evidence: snapshot.References(), Analysis: a, Policy: readiness.ExecutionPolicy{Environment: []string{"PATH"}, SetupNetwork: "unrestricted", CheckNetwork: "unrestricted", Containment: "filtered-environment"}}
 		p.Analysis.Checks = append(append([]readiness.Check(nil), p.Analysis.Checks...), o.Checks...)
 		p.NewFiles = readiness.CitedNewFiles(p.Analysis.Checks, readiness.NewScopeFiles(a.Scope, snapshot.Files))
+		if markdown := snapshot.MarkdownScopeFiles(a.Scope); len(markdown) > 0 {
+			p.MarkdownNewFiles = markdown
+		}
 		p.HarnessTimeoutMS = proposeHarnessTimeoutMS(codingBinding, len(p.Analysis.Scope))
 		if o.Policy != nil {
 			p.Policy = *o.Policy
@@ -833,6 +848,9 @@ func runIntake(ctx context.Context, o Options, key string) (r Record, err error)
 	if note, ok := newFilesNote(p); ok {
 		r.Notes = append(r.Notes, note)
 	}
+	if len(p.MarkdownNewFiles) > 0 {
+		r.Notes = append(r.Notes, fmt.Sprintf("New scope paths accepted without a component manifest by the %s: %s.", readiness.MarkdownRule, boundedPaths(p.MarkdownNewFiles, maxListedViolations)))
+	}
 	if note, ok := scopeGateSensitivePathsNote(f); ok {
 		r.Notes = append(r.Notes, note)
 	}
@@ -1479,6 +1497,9 @@ func resumeApprovedIssue(ctx context.Context, o Options, expected formula.Loaded
 		atom(&r, roles.atoms[formula.Coding], "failed", detail)
 		return finishStopped(r, detail)
 	}
+	if markdown := capturedMarkdownFiles(p, capture); len(markdown) > 0 {
+		r.Notes = append(r.Notes, fmt.Sprintf("Scope audit: captured new paths allowed by the %s: %s.", readiness.MarkdownRule, boundedPaths(markdown, maxListedViolations)))
+	}
 	// Preserve the validated report independently of capture/policy decisions.
 	phase, detail := coding.Outcome, coding.Reason
 	if phase == "completed" && len(capture.Changes) == 0 {

lab/internal/readiness/markdown_scope_test.go

@@ -0,0 +1,168 @@
+package readiness
+
+import (
+	"context"
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// markdownSnapshot is the shared repository fixture plus a tracked docs/install.md
+// and a tracked Markdown file directly in .github.
+func markdownSnapshot(t *testing.T) RepositorySnapshot {
+	t.Helper()
+	dir := sliceBFixture(t, map[string]string{"docs/install.md": "# Install\n", ".github/notes.md": "# Notes\n"})
+	s, err := Collect(context.Background(), dir, "HEAD")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return s
+}
+
+func TestMarkdownRuleAcceptsNewMarkdownNextToATrackedFile(t *testing.T) {
+	s := markdownSnapshot(t)
+	for _, file := range []string{"docs/new.md", "NEW.md", "lab/notes.md"} {
+		scope := []ScopedPath{{Path: file, Reason: "New document", Evidence: []string{"issue"}}}
+		if err := s.ValidateScope(scope); err != nil {
+			t.Fatalf("%s: %v", file, err)
+		}
+		if got := s.MarkdownScopeFiles(scope); !reflect.DeepEqual(got, []string{file}) {
+			t.Fatalf("%s: Markdown-rule files %v", file, got)
+		}
+	}
+}
+
+func TestMarkdownRuleKeepsTheManifestRuleForEveryOtherPath(t *testing.T) {
+	s := markdownSnapshot(t)
+	for _, file := range []string{"docs/new.go", "docs/new.MD", "newdir/new.md", ".github/new.md", ".github/workflows/new.md", "docs/sub/new.md", "docs/.hidden/new.md"} {
+		scope := []ScopedPath{{Path: file, Reason: "New file", Evidence: []string{"issue"}}}
+		err := s.ValidateScope(scope)
+		if err == nil || !strings.Contains(err.Error(), "needs evidence for its parent component") {
+			t.Fatalf("%s: want the manifest-rule refusal, got %v", file, err)
+		}
+		if got := s.MarkdownScopeFiles(scope); len(got) != 0 {
+			t.Fatalf("%s: listed as a Markdown-rule file: %v", file, got)
+		}
+	}
+}
+
+func TestMarkdownRuleKeepsTraversalAndDirectoryRefusals(t *testing.T) {
+	s := markdownSnapshot(t)
+	for file, want := range map[string]string{
+		"linked/new.md":        "traverses a file, symlink or submodule",
+		"README.md/new.md":     "traverses a file, symlink or submodule",
+		"docs":                 "is a directory",
+		"docs/../docs/new.md":  "",
+		".git/new.md":          "",
+		"docs/new.md\x00x.md":  "",
+		"docs/*.md":            "",
+		"docs/new.md/child.md": "needs evidence",
+		"docs/install.md/x.md": "traverses a file, symlink or submodule",
+	} {
+		scope := []ScopedPath{{Path: file, Reason: "New file", Evidence: []string{"issue"}}}
+		err := s.ValidateScope(scope)
+		if err == nil || (want != "" && !strings.Contains(err.Error(), want)) {
+			t.Fatalf("%q: want refusal %q, got %v", file, want, err)
+		}
+		if got := s.MarkdownScopeFiles(scope); len(got) != 0 {
+			t.Fatalf("%q: listed as a Markdown-rule file: %v", file, got)
+		}
+	}
+	dup := []ScopedPath{{Path: "docs/new.md", Reason: "a", Evidence: []string{"issue"}}, {Path: "docs/NEW.md", Reason: "b", Evidence: []string{"issue"}}}
+	if err := s.ValidateScope(dup); err == nil || !strings.Contains(err.Error(), "case-ambiguous") {
+		t.Fatalf("case-folded duplicate accepted: %v", err)
+	}
+}
+
+func TestMarkdownScopeFilesListsOnlyMarkdownRulePaths(t *testing.T) {
+	s := markdownSnapshot(t)
+	var manifest string
+	for _, e := range s.Evidence {
+		if e.Path == "lab/go.mod" {
+			manifest = e.ID
+		}
+	}
+	scope := []ScopedPath{
+		{Path: "README.md", Reason: "Edited", Evidence: []string{"issue"}},
+		{Path: "lab/new.go", Reason: "New code", Evidence: []string{manifest}},
+		{Path: "docs/new.md", Reason: "New document", Evidence: []string{"issue"}},
+		{Path: "NEW.md", Reason: "New document", Evidence: []string{"issue"}},
+	}
+	if err := s.ValidateScope(scope); err != nil {
+		t.Fatal(err)
+	}
+	if got := s.MarkdownScopeFiles(scope); !reflect.DeepEqual(got, []string{"docs/new.md", "NEW.md"}) {
+		t.Fatalf("Markdown-rule files %v", got)
+	}
+}
+
+func TestMarkdownRuleNeedsATrackedRegularFileInTheParent(t *testing.T) {
+	s := RepositorySnapshot{Files: []TreeFile{{Path: "docs/link", Mode: "120000"}, {Path: "docs/sub/a.md", Mode: "100644"}, {Path: "mods/m", Mode: "160000"}}}
+	for _, file := range []string{"docs/new.md", "mods/new.md", "NEW.md"} {
+		scope := []ScopedPath{{Path: file, Reason: "New document", Evidence: []string{"issue"}}}
+		if err := s.ValidateScope(scope); err == nil {
+			t.Fatalf("%s accepted next to only a symlink, submodule or nothing", file)
+		}
+	}
+	if err := s.ValidateScope([]ScopedPath{{Path: "docs/sub/new.md", Reason: "New document", Evidence: []string{"issue"}}}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func markdownPlan() Plan {
+	p := fixture()
+	p.Analysis.Scope = append(p.Analysis.Scope, ScopedPath{Path: "docs/new.md", Reason: "New document", Evidence: []string{"issue"}})
+	p.MarkdownNewFiles = []string{"docs/new.md"}
+	return p
+}
+
+func TestPlanMarkdownNewFilesJoinTheDigestAndAreOmittedWhenEmpty(t *testing.T) {
+	p := markdownPlan()
+	if err := p.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	unlisted := p
+	unlisted.MarkdownNewFiles = nil
+	a, errA := p.Digest()
+	b, errB := unlisted.Digest()
+	if errA != nil || errB != nil || a == b {
+		t.Fatalf("markdownNewFiles must join the plan digest: %v %v", errA, errB)
+	}
+	raw, err := json.Marshal(unlisted)
+	if err != nil || strings.Contains(string(raw), "markdownNewFiles") {
+		t.Fatalf("an empty markdownNewFiles must be omitted: %v %s", err, raw)
+	}
+}
+
+func TestPlanMarkdownNewFilesRejectInvalidEntries(t *testing.T) {
+	for name, mutate := range map[string]func(*Plan){
+		"outside the scope": func(p *Plan) { p.MarkdownNewFiles = []string{"docs/other.md"} },
+		"duplicate":         func(p *Plan) { p.MarkdownNewFiles = []string{"docs/new.md", "docs/new.md"} },
+		"not markdown": func(p *Plan) {
+			p.Analysis.Scope = append(p.Analysis.Scope, ScopedPath{Path: "docs/new.go", Reason: "New code", Evidence: []string{"issue"}})
+			p.MarkdownNewFiles = []string{"docs/new.go"}
+		},
+		"upper-case extension": func(p *Plan) {
+			p.Analysis.Scope = append(p.Analysis.Scope, ScopedPath{Path: "docs/new.MD", Reason: "New document", Evidence: []string{"issue"}})
+			p.MarkdownNewFiles = []string{"docs/new.MD"}
+		},
+		"dot-directory": func(p *Plan) {
+			p.Analysis.Scope = append(p.Analysis.Scope, ScopedPath{Path: ".github/new.md", Reason: "New document", Evidence: []string{"issue"}})
+			p.MarkdownNewFiles = []string{".github/new.md"}
+		},
+		"frozen evidence path": func(p *Plan) {
+			p.Evidence = append(p.Evidence, Evidence{ID: "readme", Path: "README.md", SHA256: strings.Repeat("e", 64)})
+			p.Analysis.Scope = append(p.Analysis.Scope, ScopedPath{Path: "README.md", Reason: "Edited", Evidence: []string{"readme"}})
+			p.MarkdownNewFiles = []string{"README.md"}
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			p := markdownPlan()
+			mutate(&p)
+			if err := p.Validate(); err == nil {
+				t.Fatal("an invalid Markdown-rule entry was accepted")
+			}
+		})
+	}
+}

lab/internal/readiness/plan.go

@@ -333,6 +333,11 @@ type Plan struct {
 	// and verification re-checks the base commit. It joins the plan digest; empty keeps
 	// the digest of every earlier plan unchanged.
 	NewFiles []string `json:"newFiles,omitempty"`
+	// MarkdownNewFiles lists the new scope paths intake accepted by the Markdown rule
+	// (MarkdownRule) rather than by a component manifest. The Lab derives it from the
+	// frozen tree, never from model output. It joins the plan digest; empty keeps the
+	// digest of every earlier plan unchanged.
+	MarkdownNewFiles []string `json:"markdownNewFiles,omitempty"`
 	// FullSuiteTriggers are the Formula's repository paths or globs whose change makes
 	// every selecting check run its full suite. The Lab sets them from the Formula when
 	// a check declares Select. They join the plan digest; empty keeps the digest of
@@ -726,6 +731,13 @@ func (p Plan) validateDigestible() error {
 		}
 		newFiles[file] = true
 	}
+	markdownFiles := map[string]bool{}
+	for _, file := range p.MarkdownNewFiles {
+		if !paths[file] || markdownFiles[file] || evidencePaths[file] || refs[file] || !markdownRulePath(file) {
+			return fmt.Errorf("Markdown-rule file %q must be a distinct lower-case .md scope path outside dot-directories that is not frozen evidence", file)
+		}
+		markdownFiles[file] = true
+	}
 	ids := map[string]bool{}
 	required := false
 	for _, c := range append(append([]Check{}, a.Checks...), p.Setup...) {

lab/internal/readiness/repository.go

@@ -225,6 +225,7 @@ func (s RepositorySnapshot) ValidateScope(scope []ScopedPath) error {
 	for _, f := range s.Files {
 		files[f.Path] = f
 	}
+	markdownDirs := s.trackedFileDirs()
 	refs := map[string]FileEvidence{}
 	for _, e := range s.Evidence {
 		refs[e.ID] = e
@@ -253,6 +254,9 @@ func (s RepositorySnapshot) ValidateScope(scope []ScopedPath) error {
 		if _, exists := files[file]; exists {
 			continue
 		}
+		if markdownDirs[path.Dir(file)] && markdownRulePath(file) {
+			continue
+		}
 		supported := false
 		for _, id := range item.Evidence {
 			e, ok := refs[id]
@@ -275,6 +279,81 @@ func (s RepositorySnapshot) ValidateScope(scope []ScopedPath) error {
 	return nil
 }
 
+// MarkdownRule names the reason ValidateScope accepts a new path without a
+// component manifest, for the plan, ledger and scope audit notes.
+const MarkdownRule = "Markdown rule: a new lower-case .md file outside dot-directories, next to a tracked file in the frozen snapshot"
+
+// markdownRulePath reports whether a new path's name qualifies for the Markdown
+// rule: a lower-case .md suffix and no path segment starting with '.', so
+// policy and agent configuration directories keep the manifest rule.
+func markdownRulePath(file string) bool {
+	if !strings.HasSuffix(file, ".md") {
+		return false
+	}
+	for _, part := range strings.Split(file, "/") {
+		if strings.HasPrefix(part, ".") {
+			return false
+		}
+	}
+	return true
+}
+
+// trackedFileDirs is the set of directories ("." for the root) that directly
+// hold a tracked regular file in the frozen snapshot. s.Files is bounded at
+// collection, so the set is too.
+func (s RepositorySnapshot) trackedFileDirs() map[string]bool {
+	dirs := map[string]bool{}
+	for _, f := range s.Files {
+		if f.Mode == "100644" || f.Mode == "100755" {
+			dirs[path.Dir(f.Path)] = true
+		}
+	}
+	return dirs
+}
+
+// MarkdownScopeFiles returns, in scope order, the new scope paths ValidateScope
+// accepts by the Markdown rule: valid, absent from the frozen snapshot, not
+// traversing a file, symlink or submodule, not a directory, and a Markdown-rule
+// name next to a tracked regular file. It does not validate the rest of the
+// scope; ValidateScope does that.
+func (s RepositorySnapshot) MarkdownScopeFiles(scope []ScopedPath) []string {
+	files := map[string]TreeFile{}
+	for _, f := range s.Files {
+		files[f.Path] = f
+	}
+	dirs := s.trackedFileDirs()
+	out := []string{}
+	seen := map[string]bool{}
+	for _, item := range scope {
+		file := item.Path
+		if ValidatePath(file) != nil || seen[file] {
+			continue
+		}
+		if _, exists := files[file]; exists || !markdownRulePath(file) || !dirs[path.Dir(file)] {
+			continue
+		}
+		traverses := false
+		for ancestor := path.Dir(file); ancestor != "."; ancestor = path.Dir(ancestor) {
+			if _, ok := files[ancestor]; ok {
+				traverses = true
+				break
+			}
+		}
+		for existing := range files {
+			if strings.HasPrefix(existing, file+"/") {
+				traverses = true
+				break
+			}
+		}
+		if traverses {
+			continue
+		}
+		seen[file] = true
+		out = append(out, file)
+	}
+	return out
+}
+
 // CollectForTicket selects exact paths named in ticket prose, including absent paths.
 func CollectForTicket(ctx context.Context, dir, ref, ticket string) (RepositorySnapshot, error) {
 	return CollectForTicketFor(ctx, dir, ref, ticket, CollectOptions{})