feat(search): index markdown section bodies for BM25 full-text search (#518)

internal/cbm/cbm.h

@@ -177,6 +177,11 @@ typedef enum {
     CBM_LANG_COUNT
 } CBMLanguage;
 
+// Byte budget for any prose attached to a definition as its docstring: leading
+// doc comments, and Markdown section bodies (#518). Shared with tests so the cap
+// is asserted against one definition rather than a copied literal.
+#define CBM_MAX_COMMENT_LEN 500
+
 // --- Extraction result structs ---
 
 typedef struct {

internal/cbm/extract_defs.c

@@ -16,7 +16,6 @@
 #include <ctype.h>
 
 // Buffer sizes for local arrays (base classes, params, return types).
-#define MAX_COMMENT_LEN 500
 #define MAX_BASES 16
 #define MAX_BASES_MINUS_1 15
 #define MAX_PARAMS CBM_SZ_32
@@ -1236,12 +1235,12 @@ static bool is_comment_node(const char *kind) {
             strcmp(kind, "line_comment") == 0 || strcmp(kind, "multiline_comment") == 0);
 }
 
-// Extract comment text, truncating to MAX_COMMENT_LEN.
+// Extract comment text, truncating to CBM_MAX_COMMENT_LEN.
 // #1017: snap the cut point back to a complete UTF-8 codepoint boundary.
 static char *extract_comment_text(CBMArena *a, TSNode node, const char *source) {
     char *text = cbm_node_text(a, node, source);
-    if (text && strlen(text) > MAX_COMMENT_LEN) {
-        size_t cut = MAX_COMMENT_LEN;
+    if (text && strlen(text) > CBM_MAX_COMMENT_LEN) {
+        size_t cut = CBM_MAX_COMMENT_LEN;
         while (cut > 0 && ((unsigned char)text[cut] & 0xC0) == 0x80)
             cut--;
         text[cut] = '\0';
@@ -3849,7 +3848,8 @@ static const char *qn_safe_segment(CBMArena *a, const char *name) {
     return out;
 }
 
-static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label) {
+static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label,
+                                  const char *docstring) {
     CBMArena *a = ctx->arena;
     CBMDefinition def;
     memset(&def, 0, sizeof(def));
@@ -3860,6 +3860,7 @@ static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, c
     def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET;
     def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET;
     def.is_exported = true;
+    def.docstring = docstring; // Markdown section body (#518); NULL for other configs
     cbm_defs_push(&ctx->result->defs, a, def);
 }
 
@@ -3958,6 +3959,58 @@ static char *extract_markdown_heading_name(CBMArena *a, TSNode node, const char
     return trim_heading_name(name);
 }
 
+// Capture the prose body beneath a Markdown heading so BM25 can search the content
+// and not just the heading text (#518). In the tree-sitter-markdown grammar each
+// heading lives inside a `section` node that also holds the body blocks and any
+// nested subsections; the body is the source span between the heading and either
+// the first nested subsection or the end of the section. Nested subsections are
+// excluded because each gets its own Section node and its own body. Returns NULL
+// when there is no enclosing section or no body text. Trimmed, and capped at
+// CBM_MAX_COMMENT_LEN (the same budget docstrings use) without splitting a UTF-8
+// sequence.
+static char *extract_markdown_section_body(CBMArena *a, TSNode heading, const char *source) {
+    TSNode parent = ts_node_parent(heading);
+    if (ts_node_is_null(parent) || strcmp(ts_node_type(parent), "section") != 0) {
+        return NULL;
+    }
+    uint32_t body_start = ts_node_end_byte(heading);
+    uint32_t body_end = ts_node_end_byte(parent);
+    // Stop at the first nested subsection — it gets its own Section node + body.
+    uint32_t cc = ts_node_child_count(parent);
+    for (uint32_t i = 0; i < cc; i++) {
+        TSNode ch = ts_node_child(parent, i);
+        if (ts_node_start_byte(ch) >= body_start && strcmp(ts_node_type(ch), "section") == 0) {
+            body_end = ts_node_start_byte(ch);
+            break;
+        }
+    }
+    // Trim surrounding whitespace/newlines. UTF-8 lead and continuation bytes are all
+    // >= 0x80, so a byte-wise <= ' ' test never cuts a multi-byte character.
+    while (body_start < body_end && (unsigned char)source[body_start] <= ' ') {
+        body_start++;
+    }
+    while (body_end > body_start && (unsigned char)source[body_end - 1] <= ' ') {
+        body_end--;
+    }
+    if (body_end <= body_start) {
+        return NULL;
+    }
+    size_t len = (size_t)(body_end - body_start);
+    if (len > CBM_MAX_COMMENT_LEN) {
+        len = CBM_MAX_COMMENT_LEN;
+        // Back off so the cap never splits a UTF-8 multi-byte sequence: source[start+len]
+        // is the first excluded byte, and a continuation byte there means we landed
+        // mid-character.
+        while (len > 0 && ((unsigned char)source[body_start + len] & 0xC0) == 0x80) {
+            len--;
+        }
+        if (len == 0) {
+            return NULL;
+        }
+    }
+    return cbm_arena_strndup(a, source + body_start, len);
+}
+
 // INI: extract section name from section node.
 static char *find_ini_section_name(CBMArena *a, TSNode node, const char *source) {
     uint32_t nc = ts_node_child_count(node);
@@ -4021,6 +4074,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
     CBMArena *a = ctx->arena;
     char *name = NULL;
     const char *label = "Class";
+    const char *docstring = NULL;
 
     if (ctx->language == CBM_LANG_TOML &&
         (strcmp(kind, "table") == 0 || strcmp(kind, "table_array_element") == 0)) {
@@ -4036,14 +4090,15 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
         // label rather than degrade it to match a test. The markdown repro asserts
         // "Class"; that assertion is the inaccurate side and is flagged for review.
         label = "Section";
+        docstring = extract_markdown_section_body(a, node, ctx->source); // #518
     } else if (ctx->language == CBM_LANG_HCL && strcmp(kind, "block") == 0) {
         name = find_hcl_block_name(a, node, ctx->source);
     } else {
         return false;
     }
 
     if (name && name[0]) {
-        push_simple_class_def(ctx, node, name, label);
+        push_simple_class_def(ctx, node, name, label, docstring);
     }
     return true;
 }
@@ -4098,7 +4153,7 @@ static bool extract_sql_ddl_class_def(CBMExtractCtx *ctx, TSNode node, const cha
     if (!name || !name[0]) {
         return false;
     }
-    push_simple_class_def(ctx, node, name, label);
+    push_simple_class_def(ctx, node, name, label, NULL); // SQL tables/views carry no body
     // Must match push_simple_class_def's QN exactly (qn_safe_segment included)
     // or pass_usages cannot find the enclosing def for the lineage source.
     const char *qn =