From 117736d615086b8a5b7529e919ae0fe1d98da25b Mon Sep 17 00:00:00 2001 From: Viktor Palmkvist Date: Tue, 25 Aug 2026 13:56:31 +0200 Subject: [PATCH] Refactor scripts, generate reference documentation --- .github/workflows/gh-pages.yml | 6 + docs/.gitignore | 5 +- package.json | 3 +- .../check-changelog.fish | 15 +- scripts/gen-reference-docs.fish | 65 +++++++ scripts/reference-doc-gen.awk | 168 ++++++++++++++++++ scripts/repo-utils.fish | 17 ++ 7 files changed, 264 insertions(+), 15 deletions(-) rename check-changelog.fish => scripts/check-changelog.fish (93%) create mode 100755 scripts/gen-reference-docs.fish create mode 100644 scripts/reference-doc-gen.awk create mode 100644 scripts/repo-utils.fish diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index b441469..6b7c691 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -28,8 +28,14 @@ jobs: uses: actions/configure-pages@v3 - name: Install dependencies run: "npm install" + - name: Install fish + run: | + sudo apt-get update + sudo apt-get install -y fish - name: Pull models from TreePPL repo run: "npm run pull-treeppl-models" + - name: Generate reference documentation + run: "npm run gen-reference-docs" - name: Build with Docusaurus run: "npm run build" - name: Upload artifact diff --git a/docs/.gitignore b/docs/.gitignore index 0854cf8..01f5e65 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,2 +1,5 @@ *-remote.md -!_*-remote.md \ No newline at end of file +!_*-remote.md + +# Generated by scripts/gen-reference-docs.fish +/Reference/ \ No newline at end of file diff --git a/package.json b/package.json index 721ccc7..06d61df 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids", "typecheck": "tsc", - "pull-treeppl-models": "./pull_md.sh" + "pull-treeppl-models": "./pull_md.sh", + "gen-reference-docs": "./scripts/gen-reference-docs.fish" }, "dependencies": { "@docusaurus/core": "^3.8.1", diff --git a/check-changelog.fish b/scripts/check-changelog.fish similarity index 93% rename from check-changelog.fish rename to scripts/check-changelog.fish index e021494..ba28db2 100755 --- a/check-changelog.fish +++ b/scripts/check-changelog.fish @@ -2,22 +2,11 @@ # === Library stuff === -set -g repoDir (path resolve (status dirname)) +source (path resolve (status dirname))/repo-utils.fish set -g repoNames set -g repoUrls -set -g online yes - -function ensureRepo --argument-names repoName url - set -l dir "$repoDir/other-repositories/$repoName" - if not test -d $dir - git clone $url $dir || exit 1 - end - cd $dir - set -qg online && begin; git pull --force || exit 1; end -end - function prsSince --argument-names repoName date set -l dir "$repoDir/other-repositories/$repoName" cd $dir @@ -154,7 +143,7 @@ addRepo treeppl https://github.com/treeppl/treeppl.git addRepo treeppl-python https://github.com/treeppl/treeppl-python.git addRepo treepplr https://github.com/treeppl/treepplr.git -splitOffBullets (begin; unmentionedPRs $startDate $repoDir/docs/changelog.md $repoDir/(status basename); string join \n -- $extraPRs; end) | formatLinks +splitOffBullets (begin; unmentionedPRs $startDate $repoDir/docs/changelog.md $repoDir/scripts/(status basename); string join \n -- $extraPRs; end) | formatLinks echo showBullets | formatLinks diff --git a/scripts/gen-reference-docs.fish b/scripts/gen-reference-docs.fish new file mode 100755 index 0000000..f2c74f9 --- /dev/null +++ b/scripts/gen-reference-docs.fish @@ -0,0 +1,65 @@ +#!/usr/bin/env fish + +# Generates docs/Reference/**/*.md from the .tppl sources under +# other-repositories/treeppl/lib. Each generated file lists the types, +# functions, and type aliases defined in the corresponding .tppl file, +# sorted alphabetically, along with the comment preceding each one (if +# any). docs/Reference is git-ignored; run this script to (re)generate it. + +source (path resolve (status dirname))/repo-utils.fish + +set -l scriptDir $repoDir/scripts + +ensureRepo treeppl https://github.com/treeppl/treeppl.git + +set -l libDir $repoDir/other-repositories/treeppl/lib +set -l outDir $repoDir/docs/Reference + +set -l US (printf '\x01') +set -l NL (printf '\x02') + +rm -rf $outDir +mkdir -p $outDir + +set -l count 0 +for src in (find $libDir -name '*.tppl' | sort) + set count (math $count + 1) + set -l rel (string replace -- "$libDir/" "" $src) + set -l out $outDir/(string replace -r '\.tppl$' '.md' -- $rel) + mkdir -p (path dirname $out) + + set -l entries (awk -f $scriptDir/reference-doc-gen.awk $src | sort -f) + + begin + echo "# "(path basename $src) + echo + + if test (count $entries) -eq 0 + echo "No definitions found in this file." + else + for entry in $entries + set -l fields (string split -- $US $entry) + set -l name $fields[1] + set -l kind $fields[2] + set -l sigLines (string split -- $NL $fields[3]) + set -l commentLines + if test -n "$fields[4]" + set commentLines (string split -- $NL $fields[4]) + end + + echo "## _"$kind"_ `"$name"`" + echo + if test (count $commentLines) -gt 0 + printf '%s\n' $commentLines + echo + end + echo '```tppl' + printf '%s\n' $sigLines + echo '```' + echo + end + end + end > $out +end + +echo "Generated reference docs for $count file(s) in $outDir" diff --git a/scripts/reference-doc-gen.awk b/scripts/reference-doc-gen.awk new file mode 100644 index 0000000..5804df5 --- /dev/null +++ b/scripts/reference-doc-gen.awk @@ -0,0 +1,168 @@ +# Extracts top-level type, type alias, and function definitions (including +# `model function`) from a .tppl source file, together with the comment +# block immediately preceding each one, if any. +# +# Emits one record per line to stdout: name US kind US signature US comment +# where US is \x01 and internal newlines within signature/comment are \x02, +# so each record stays on a single physical output line (sortable, and easy +# to split back apart by the caller). + +BEGIN { + US = sprintf("%c", 1) + NLc = sprintf("%c", 2) + OFS = US + depth = 0 + capturing = 0 + mode = "" + haveComment = 0 + commentBuf = "" + inBlock = 0 +} + +function resetComment() { + commentBuf = "" + haveComment = 0 +} + +function addCommentLine(line, cleaned) { + cleaned = line + sub(/^[ \t]*\/\*/, "", cleaned) + sub(/^[ \t]*\/\//, "", cleaned) + sub(/\*\/[ \t]*$/, "", cleaned) + sub(/^[ \t]*\*[ \t]?/, "", cleaned) + gsub(/^[ \t]+/, "", cleaned) + gsub(/[ \t]+$/, "", cleaned) + if (cleaned == "") return + if (haveComment) commentBuf = commentBuf NLc cleaned + else { commentBuf = cleaned; haveComment = 1 } +} + +function braceDelta(line, i, c, d) { + d = 0 + for (i = 1; i <= length(line); i++) { + c = substr(line, i, 1) + if (c == "{") d++ + else if (c == "}") d-- + } + return d +} + +function emit() { + print name, kind, sigBuf, (haveComment ? commentBuf : "") + resetComment() + capturing = 0 + mode = "" + sigBuf = "" +} + +# Consumes one line of a function's header (the part before its body's +# opening brace). Once the brace is found, only the text before it is kept +# in sigBuf, and capturing switches to silently tracking depth through the +# body via "func-body-skip". +function headerLine(line, pos, prefix) { + pos = index(line, "{") + if (pos == 0) { + sigBuf = (sigBuf == "" ? line : sigBuf NLc line) + return + } + prefix = substr(line, 1, pos - 1) + gsub(/[ \t]+$/, "", prefix) + if (prefix != "") sigBuf = (sigBuf == "" ? prefix : sigBuf NLc prefix) + depth = braceDelta(line) + mode = "func-body-skip" + if (depth <= 0) emit() +} + +{ + line = $0 + + if (capturing) { + if (mode == "variant") { + t = line + gsub(/^[ \t]+/, "", t) + gsub(/[ \t]+$/, "", t) + if (substr(t, 1, 1) == "|") { + sigBuf = sigBuf NLc line + next + } else { + emit() + # fall through: reprocess this line as top-level below + } + } else if (mode == "func-header") { + headerLine(line) + next + } else if (mode == "func-body-skip") { + depth += braceDelta(line) + if (depth <= 0) emit() + next + } else { + sigBuf = sigBuf NLc line + depth += braceDelta(line) + if (depth <= 0) emit() + next + } + } + + t = line + gsub(/^[ \t]+/, "", t) + gsub(/[ \t]+$/, "", t) + + if (inBlock) { + addCommentLine(line) + if (t ~ /\*\/[ \t]*$/) inBlock = 0 + next + } + + if (t == "") { resetComment(); next } + + if (t ~ /^\/\//) { addCommentLine(line); next } + + if (t ~ /^\/\*/) { + addCommentLine(line) + if (t !~ /\*\/[ \t]*$/) inBlock = 1 + next + } + + if (match(t, /^(model[ \t]+)?type[ \t]+alias[ \t]+[A-Za-z_][A-Za-z0-9_]*/)) { + kind = "type alias" + name = t + sub(/^(model[ \t]+)?type[ \t]+alias[ \t]+/, "", name) + sub(/[^A-Za-z0-9_].*$/, "", name) + } else if (match(t, /^(model[ \t]+)?type[ \t]+[A-Za-z_][A-Za-z0-9_]*/)) { + kind = "type" + name = t + sub(/^(model[ \t]+)?type[ \t]+/, "", name) + sub(/[^A-Za-z0-9_].*$/, "", name) + } else if (match(t, /^(model[ \t]+)?function[ \t]+[A-Za-z_][A-Za-z0-9_]*/)) { + kind = "function" + name = t + sub(/^(model[ \t]+)?function[ \t]+/, "", name) + sub(/[^A-Za-z0-9_].*$/, "", name) + } else { + resetComment() + next + } + + sigBuf = "" + if (kind == "type" && braceDelta(line) == 0 && t ~ /=[ \t]*$/) { + sigBuf = line + mode = "variant" + capturing = 1 + next + } + if (kind == "function") { + mode = "func-header" + capturing = 1 + headerLine(line) + next + } + + sigBuf = line + depth = braceDelta(line) + if (depth <= 0) { + emit() + } else { + mode = "brace" + capturing = 1 + } +} diff --git a/scripts/repo-utils.fish b/scripts/repo-utils.fish new file mode 100644 index 0000000..2318d41 --- /dev/null +++ b/scripts/repo-utils.fish @@ -0,0 +1,17 @@ +# Shared helpers for downloading external repositories and keeping them +# up to date. Repositories are cloned into other-repositories/ at +# the project root (which is git-ignored), and left in place between runs +# so subsequent calls only need to pull. + +set -g repoDir (path resolve (status dirname)/..) + +set -g online yes + +function ensureRepo --argument-names repoName url + set -l dir "$repoDir/other-repositories/$repoName" + if not test -d $dir + git clone $url $dir || exit 1 + end + cd $dir + set -qg online && begin; git pull --force || exit 1; end +end