diff --git a/scripts/frontmatter.awk b/scripts/frontmatter.awk index c741fc94..68420a0f 100644 --- a/scripts/frontmatter.awk +++ b/scripts/frontmatter.awk @@ -1,5 +1,32 @@ # Observe the supported scalar/mapping header shape, never claim full YAML validation. function trim(s) { sub(/^[[:space:]]+/,"",s); sub(/[[:space:]]+$/,"",s); return s } +# Normalize the same Unicode whitespace observed in escaped scalars for presence +# only. Preserve the original scalar for provenance; use complete UTF-8 strings +# so the C and UTF-8 locales agree without partial-byte regex ranges. +function nonblank_text(s,spaces,blanks,n,i) { + spaces="\302\205 \302\240 \341\232\200 " \ + "\342\200\200 \342\200\201 \342\200\202 \342\200\203 \342\200\204 \342\200\205 " \ + "\342\200\206 \342\200\207 \342\200\210 \342\200\211 \342\200\212 \342\200\213 " \ + "\342\200\250 \342\200\251 \342\200\257 \342\201\237 \343\200\200" + n=split(spaces,blanks,"[ ]") + for (i=1;i<=n;i++) gsub(blanks[i]," ",s) + return trim(s) != "" +} +# Observe literal controls before the scalar decoder can mark them as text. +# Match complete UTF-8 strings: byte ranges inside a regex character class are +# invalid collation characters in GNU awk's UTF-8 locales. Allowed YAML +# whitespace is not a forbidden control. +function forbidden_control(s, t,controls,c1,n,i) { + t=s; gsub(/[\t\r]/,"",t); gsub("\302\205","",t) + if (t ~ /[[:cntrl:]]/) return 1 + controls="\302\200 \302\201 \302\202 \302\203 \302\204 " \ + "\302\206 \302\207 \302\210 \302\211 \302\212 \302\213 \302\214 \302\215 \302\216 \302\217 " \ + "\302\220 \302\221 \302\222 \302\223 \302\224 \302\225 \302\226 \302\227 " \ + "\302\230 \302\231 \302\232 \302\233 \302\234 \302\235 \302\236 \302\237" + n=split(controls,c1," ") + for (i=1;i<=n;i++) if (index(s,c1[i])) return 1 + return 0 +} # A presence observer, not a general YAML decoder: normalize escaped whitespace, # decode printable ASCII, and retain valid non-ASCII escapes as nonblank text. function quoted_text(s,q, i,c,n,hex,j,d,code,out) { @@ -10,7 +37,7 @@ function quoted_text(s,q, i,c,n,hex,j,d,code,out) { if (c != "\\") { out=out c; continue } c=substr(s,++i,1) if (c == "\\" || c == "\"" || c == "/") { out=out c; continue } - if (c ~ /^[0abe]$/) { out=out "\\" c; continue } + if (c ~ /^[0abe]$/) return "" if (c ~ /^[tnvfrN_LP ]$/ || c == "\t") { out=out " "; continue } if (c != "x" && c != "u" && c != "U") return "" n=(c == "x" ? 2 : (c == "u" ? 4 : 8)); hex=substr(s,i+1,n) @@ -20,8 +47,8 @@ function quoted_text(s,q, i,c,n,hex,j,d,code,out) { if (code > 1114111 || (code >= 55296 && code <= 57343)) return "" if ((code >= 9 && code <= 13) || code == 32 || code == 133 || code == 160 || code == 5760 || (code >= 8192 && code <= 8202) || code == 8232 || code == 8233 || - code == 8239 || code == 8287 || code == 12288) out=out " " - else if (code < 32 || (code >= 127 && code <= 159)) out=out "\\" c hex + code == 8203 || code == 8239 || code == 8287 || code == 12288) out=out " " + else if (code < 32 || (code >= 127 && code <= 159)) return "" else if (code < 127) out=out sprintf("%c",code) else out=out "\\" c hex i+=n @@ -40,7 +67,7 @@ function scalar(s, q,i,c,escaped,tail) { tail=trim(substr(s,i+1)) if (tail != "" && substr(tail,1,1) != "#") return "" s=quoted_text(substr(s,2,i-2),q) - scalar_ok=(quoted_ok && trim(s) != ""); return s + scalar_ok=(quoted_ok && nonblank_text(s)); return s } escaped=0 } @@ -53,14 +80,15 @@ function scalar(s, q,i,c,escaped,tail) { s ~ /^[-+]?0([xX][0-9a-fA-F_]+|[oO][0-7_]+|[bB][01_]+)$/ || s ~ /^[-+]?\.([iI][nN][fF]|[nN][aA][nN])$/ || s ~ /^[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+(\.[0-9_]*)?$/) return "" - scalar_ok=1; return s + scalar_ok=nonblank_text(s); return s } { sub(/\r$/,"") + if (forbidden_control($0)) bad=1 if (NR == 1) { if ($0 !~ /^---[[:space:]]*$/) bad=1; next } if ($0 ~ /^---[[:space:]]*$/) { closed=1; exit } if (bad) next - if (mode == "text" && block && $0 ~ /^[[:space:]]/ && $0 ~ /[^[:space:]]/) found=1 + if (mode == "text" && block && $0 ~ /^[[:space:]]/ && nonblank_text($0)) found=1 if ($0 ~ /^[[:space:]]*(#.*)?$/) next if ($0 ~ /^[A-Za-z_][A-Za-z0-9_-]*:/) { key=$0; sub(/:.*/,"",key) diff --git a/scripts/validate-manifests.sh b/scripts/validate-manifests.sh index 1a9c28bb..46db6e77 100755 --- a/scripts/validate-manifests.sh +++ b/scripts/validate-manifests.sh @@ -229,7 +229,7 @@ validate_marketplace_renames() { # or a 'url' (remote transport). validate_mcp_json() { local mcp="$1" document - if ! document=$(cat "$mcp") || ! jq -es 'length == 1 and (.[0] | type == "object")' <<< "$document" > /dev/null 2>&1; then + if ! document=$(cat -- "$mcp" | json_source_retain) || ! jq -es 'length == 1 and (.[0] | type == "object")' <<< "$document" > /dev/null 2>&1; then echo "::error::$mcp: not valid JSON" return 1 fi @@ -257,16 +257,24 @@ validate_mcp_json() { fi if ! jq -e ' def nonblank: type == "string" and test("[^[:space:]]"); - def string_map: type == "object" and all(to_entries[]; (.key|nonblank) and (.value|type)=="string"); + def process_text: type == "string" and (contains("\u0000") | not); + def environment: type == "object" and all(to_entries[]; + (.key | nonblank and process_text and (contains("=") | not)) and (.value | process_text)); + def header_name: nonblank and (explode | all(.[]; + (. >= 48 and . <= 57) or (. >= 65 and . <= 90) or (. >= 97 and . <= 122) or + (. as $code | [33,35,36,37,38,39,42,43,45,46,94,95,96,124,126] | index($code) != null))); + def header_value: type == "string" and (explode | all(.[]; + . == 9 or (. >= 32 and . <= 126) or (. >= 128 and . <= 255))); + def headers: type == "object" and all(to_entries[]; (.key|header_name) and (.value|header_value)); all(.mcpServers | to_entries[]; (.key | nonblank) and (.value | type == "object" and (if has("command") then - (.command | nonblank) and (has("url") | not) and + (.command | nonblank and process_text) and (has("url") | not) and (if has("type") then .type == "stdio" else true end) else (.url | nonblank) and (.type == "http" or .type == "sse") end) and - (if has("args") then (.args | type == "array" and all(.[]; type == "string")) else true end) and - (if has("env") then (.env | string_map) else true end) and - (if has("headers") then (.headers | string_map) else true end))) + (if has("args") then (.args | type == "array" and all(.[]; process_text)) else true end) and + (if has("env") then (.env | environment) else true end) and + (if has("headers") then (.headers | headers) else true end))) ' <<< "$document" > /dev/null; then echo "::error::$mcp: invalid or missing a 'command' (stdio) or 'url' (remote), transport, arguments, environment or headers" return 1 diff --git a/scripts/validate-manifests.test.sh b/scripts/validate-manifests.test.sh index 4ea5e513..37d83b12 100755 --- a/scripts/validate-manifests.test.sh +++ b/scripts/validate-manifests.test.sh @@ -2045,7 +2045,7 @@ for header in unclosed duplicate-name duplicate-description null bool number seq done # Escaped agent identities must be valid text after YAML escape interpretation. for field in name description; do - for value in '"\n"' '"\t\r "' '"\x20"' '"\u0020"' '"\U00000020"' '"\q"' '"\xG0"' '"\u123"' '"\U00110000"' '"\uD800"'; do + for value in '"\0"' '"Text\0value"' '"\a"' '"\b"' '"\e"' '"\x00"' '"\u0001"' '"\U0000007f"' '"\u009f"' '"\u200B"' '"\U0000200b"' '"\n"' '"\t\r "' '"\x20"' '"\u0020"' '"\U00000020"' '"\q"' '"\xG0"' '"\u123"' '"\U00110000"' '"\uD800"'; do d=$(fresh); mkdir -p "$d/plugins/alpha/agents" f="$d/plugins/alpha/agents/sample.agent.md" printf '%s\n' '---' 'name: sample' 'description: A real description.' > "$f" @@ -2070,6 +2070,172 @@ for field in name description; do done done +# The header observer must compile and preserve non-ASCII text in both byte and +# UTF-8 locales. GNU awk rejects regex ranges made from partial UTF-8 bytes. +for parser_locale in C C.UTF-8; do + d=$(fresh); mkdir -p "$d/plugins/alpha/agents" + printf '%s\n' '---' 'name: sample' 'description: Unicode café 日本語.' '---' body \ + > "$d/plugins/alpha/agents/sample.agent.md" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `sample` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + LC_ALL="$parser_locale" check_pass "non-ASCII agent identity survives $parser_locale parser" "$d" +done + +# Literal control bytes must be refused too, including when catalogue omission +# would otherwise conceal a parse failure after Bash erased the original NUL. +for field in name description; do + for encoding in '\001' '\177' '\302\237'; do + for quoting in double single plain block; do + d=$(fresh); mkdir -p "$d/plugins/alpha/agents" + f="$d/plugins/alpha/agents/sample.agent.md" + printf '%s\n' '---' 'name: sample' 'description: A real description.' > "$f" + sed "/^$field:/d" "$f" > "$d/header"; mv "$d/header" "$f" + case $quoting in + double) printf '%s: "%b"\n' "$field" "$encoding" >> "$f" ;; + single) printf "%s: '%b'\n" "$field" "$encoding" >> "$f" ;; + plain) printf '%s: %b\n' "$field" "$encoding" >> "$f" ;; + block) printf '%s: |\n %b\n' "$field" "$encoding" >> "$f" ;; + esac + printf '%s\n' '---' body >> "$f" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `sample` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + check_fail "literal control $encoding in $quoting $field is refused" "must declare a non-empty '$field'" "$d" + done + done +done +d=$(fresh) +printf '{"mcpServers":{"fixture":{"command":"to\000ol"}}}\n' > "$d/plugins/alpha/.mcp.json" +check_fail 'original MCP NUL cannot vanish before validation' 'not valid JSON' "$d" +for field in name description; do + for encoding in '\013' '\014'; do + d=$(fresh); mkdir -p "$d/plugins/alpha/agents" + f="$d/plugins/alpha/agents/sample.agent.md" + printf '%s\n' '---' 'name: sample' 'description: A real description.' > "$f" + sed "/^$field:/d" "$f" > "$d/header"; mv "$d/header" "$f" + printf '%s: "Visible%btext"\n' "$field" "$encoding" >> "$f" + printf '%s\n' '---' body >> "$f" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `sample` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + check_fail "raw YAML control $encoding in $field is refused" "must declare a non-empty '$field'" "$d" + done +done + +# Delimiters participate in the same original-byte header observation. +for delimiter in opening closing; do + for encoding in '\013' '\014'; do + d=$(fresh); mkdir -p "$d/plugins/alpha/agents" + f="$d/plugins/alpha/agents/sample.agent.md" + if [ "$delimiter" = opening ]; then printf '%s%b\n' '---' "$encoding" > "$f" + else printf '%s\n' '---' > "$f"; fi + printf '%s\n' 'name: sample' 'description: A visible description.' >> "$f" + if [ "$delimiter" = closing ]; then printf '%s%b\n' '---' "$encoding" >> "$f" + else printf '%s\n' '---' >> "$f"; fi + printf '%s\n' body >> "$f" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `sample` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + check_fail "control $encoding on $delimiter delimiter is refused" "must declare a non-empty 'name'" "$d" + done +done + +# A zero-width-space-only scalar is not usable identity text. Literal escape +# spelling and visible text adjacent to the character are still real content. +for field in name description; do + for quoting in double single plain block; do + d=$(fresh); mkdir -p "$d/plugins/alpha/agents" + f="$d/plugins/alpha/agents/sample.agent.md" + printf '%s\n' '---' 'name: sample' 'description: A visible description.' > "$f" + sed "/^$field:/d" "$f" > "$d/header"; mv "$d/header" "$f" + case $quoting in + double) printf '%s: "\342\200\213"\n' "$field" >> "$f" ;; + single) printf "%s: '\342\200\213'\n" "$field" >> "$f" ;; + plain) printf '%s: \342\200\213\n' "$field" >> "$f" ;; + block) printf '%s: |\n \342\200\213\n' "$field" >> "$f" ;; + esac + printf '%s\n' '---' body >> "$f" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `sample` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + check_fail "zero-width-only $quoting $field is refused" "must declare a non-empty '$field'" "$d" + done + for value in '"Visible\u200Bé"' '"\\u200B"'; do + d=$(fresh); mkdir -p "$d/plugins/alpha/agents" + f="$d/plugins/alpha/agents/sample.agent.md" + printf '%s\n' '---' 'name: sample' 'description: A visible description.' > "$f" + sed "/^$field:/d" "$f" > "$d/header"; mv "$d/header" "$f" + printf '%s: %s\n' "$field" "$value" >> "$f" + printf '%s\n' '---' body >> "$f" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `sample` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + check_pass "visible or literal escape $field $value stays usable" "$d" + done +done + +# Literal whitespace follows the already supported escaped-codepoint policy in +# every locale. The C locale must not mistake UTF-8 bytes for visible content. +for field in name description; do + for encoding in '\302\205' '\302\240' '\341\232\200' \ + '\342\200\200' '\342\200\201' '\342\200\202' '\342\200\203' \ + '\342\200\204' '\342\200\205' '\342\200\206' '\342\200\207' \ + '\342\200\210' '\342\200\211' '\342\200\212' \ + '\342\200\250' '\342\200\251' '\342\200\257' '\342\201\237' '\343\200\200'; do + d=$(fresh); mkdir -p "$d/plugins/alpha/agents" + f="$d/plugins/alpha/agents/sample.agent.md" + printf '%s\n' '---' 'name: sample' 'description: A visible description.' > "$f" + sed "/^$field:/d" "$f" > "$d/header"; mv "$d/header" "$f" + printf '%s: "%b"\n' "$field" "$encoding" >> "$f" + printf '%s\n' '---' body >> "$f" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `sample` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + LC_ALL=C check_fail "literal Unicode space $encoding in $field is blank in C" "must declare a non-empty '$field'" "$d" + done + for parser_locale in C C.UTF-8; do + d=$(fresh); mkdir -p "$d/plugins/alpha/agents" + f="$d/plugins/alpha/agents/sample.agent.md" + printf '%s\n' '---' 'name: sample' 'description: A visible description.' > "$f" + sed "/^$field:/d" "$f" > "$d/header"; mv "$d/header" "$f" + printf '%s: "Visible\302\240Unicode"\n' "$field" >> "$f" + printf '%s\n' '---' body >> "$f" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `sample` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + LC_ALL="$parser_locale" check_pass "visible Unicode space in $field stays usable in $parser_locale" "$d" + done +done + +# Transport declarations must survive native process/HTTP argument validation. +for server in \ + '{"command":"node\u0000"}' \ + '{"command":"node","args":["\u0000"]}' \ + '{"command":"node","env":{"TOKEN":"x\u0000"}}' \ + '{"command":"node","env":{"BAD=KEY":"x"}}' \ + '{"type":"http","url":"https://example.com/mcp","headers":{"Bad:Name":"value"}}' \ + '{"type":"http","url":"https://example.com/mcp","headers":{"Authorization":"value\r\nnext"}}' \ + '{"type":"http","url":"https://example.com/mcp","headers":{"Authorization":"\u0100"}}'; do + d=$(fresh) + printf '{"mcpServers":{"fixture":%s}}\n' "$server" > "$d/plugins/alpha/.mcp.json" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `fixture` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + check_fail 'unusable MCP wire values are refused' 'invalid or missing' "$d" +done +# shellcheck disable=SC2016 # Variable references are literal packaged values. +for server in \ + '{"command":"命令","args":["","${TOKEN}","é"],"env":{"TOKEN":""}}' \ + '{"type":"http","url":"https://example.com/mcp","headers":{"X-Token":"${TOKEN}","X-Empty":"","X-Tab":"a\tb","X-Latin":"é"}}'; do + d=$(fresh) + printf '{"mcpServers":{"fixture":%s}}\n' "$server" > "$d/plugins/alpha/.mcp.json" + # shellcheck disable=SC2016 # Literal catalogue tokens. + sed 's/`example-skill` | Alpha plugin/`example-skill`, `fixture` | Alpha plugin/' "$d/docs/plugins.md" > "$d/table" + mv "$d/table" "$d/docs/plugins.md" + check_pass 'supported MCP wire values stay usable' "$d" +done + for owner in null malformed nested duplicate-owner duplicate-metadata unclosed; do d=$(fresh) f="$d/plugins/alpha/skills/example-skill/SKILL.md"