Refactor scripts, generate reference documentation

.github/workflows/gh-pages.yml

@@ -28,8 +28,14 @@ jobs:
         uses: actions/configure-pages@v3
       - name: Install dependencies
         run: "npm install"
+      - name: Install fish
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y fish
       - name: Pull models from TreePPL repo
         run: "npm run pull-treeppl-models"
+      - name: Generate reference documentation
+        run: "npm run gen-reference-docs"
       - name: Build with Docusaurus
         run: "npm run build"
       - name: Upload artifact

docs/.gitignore

@@ -1,2 +1,5 @@
 *-remote.md
-!_*-remote.md
+!_*-remote.md
+
+# Generated by scripts/gen-reference-docs.fish
+/Reference/

package.json

@@ -13,7 +13,8 @@
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
     "typecheck": "tsc",
-    "pull-treeppl-models": "./pull_md.sh"
+    "pull-treeppl-models": "./pull_md.sh",
+    "gen-reference-docs": "./scripts/gen-reference-docs.fish"
   },
   "dependencies": {
     "@docusaurus/core": "^3.8.1",

check-changelog.fish → scripts/check-changelog.fish

@@ -2,22 +2,11 @@
 
 # === Library stuff ===
 
-set -g repoDir (path resolve (status dirname))
+source (path resolve (status dirname))/repo-utils.fish
 
 set -g repoNames
 set -g repoUrls
 
-set -g online yes
-
-function ensureRepo --argument-names repoName url
-    set -l dir "$repoDir/other-repositories/$repoName"
-    if not test -d $dir
-        git clone $url $dir || exit 1
-    end
-    cd $dir
-    set -qg online && begin; git pull --force || exit 1; end
-end
-
 function prsSince --argument-names repoName date
     set -l dir "$repoDir/other-repositories/$repoName"
     cd $dir
@@ -154,7 +143,7 @@ addRepo treeppl https://github.com/treeppl/treeppl.git
 addRepo treeppl-python https://github.com/treeppl/treeppl-python.git
 addRepo treepplr https://github.com/treeppl/treepplr.git
 
-splitOffBullets (begin; unmentionedPRs $startDate $repoDir/docs/changelog.md $repoDir/(status basename); string join \n -- $extraPRs; end) | formatLinks
+splitOffBullets (begin; unmentionedPRs $startDate $repoDir/docs/changelog.md $repoDir/scripts/(status basename); string join \n -- $extraPRs; end) | formatLinks
 echo
 showBullets | formatLinks
 

scripts/gen-reference-docs.fish

@@ -0,0 +1,65 @@
+#!/usr/bin/env fish
+
+# Generates docs/Reference/**/*.md from the .tppl sources under
+# other-repositories/treeppl/lib. Each generated file lists the types,
+# functions, and type aliases defined in the corresponding .tppl file,
+# sorted alphabetically, along with the comment preceding each one (if
+# any). docs/Reference is git-ignored; run this script to (re)generate it.
+
+source (path resolve (status dirname))/repo-utils.fish
+
+set -l scriptDir $repoDir/scripts
+
+ensureRepo treeppl https://github.com/treeppl/treeppl.git
+
+set -l libDir $repoDir/other-repositories/treeppl/lib
+set -l outDir $repoDir/docs/Reference
+
+set -l US (printf '\x01')
+set -l NL (printf '\x02')
+
+rm -rf $outDir
+mkdir -p $outDir
+
+set -l count 0
+for src in (find $libDir -name '*.tppl' | sort)
+    set count (math $count + 1)
+    set -l rel (string replace -- "$libDir/" "" $src)
+    set -l out $outDir/(string replace -r '\.tppl$' '.md' -- $rel)
+    mkdir -p (path dirname $out)
+
+    set -l entries (awk -f $scriptDir/reference-doc-gen.awk $src | sort -f)
+
+    begin
+        echo "# "(path basename $src)
+        echo
+
+        if test (count $entries) -eq 0
+            echo "No definitions found in this file."
+        else
+            for entry in $entries
+                set -l fields (string split -- $US $entry)
+                set -l name $fields[1]
+                set -l kind $fields[2]
+                set -l sigLines (string split -- $NL $fields[3])
+                set -l commentLines
+                if test -n "$fields[4]"
+                    set commentLines (string split -- $NL $fields[4])
+                end
+
+                echo "## _"$kind"_ `"$name"`"
+                echo
+                if test (count $commentLines) -gt 0
+                    printf '%s\n' $commentLines
+                    echo
+                end
+                echo '```tppl'
+                printf '%s\n' $sigLines
+                echo '```'
+                echo
+            end
+        end
+    end > $out
+end
+
+echo "Generated reference docs for $count file(s) in $outDir"

scripts/reference-doc-gen.awk

@@ -0,0 +1,168 @@
+# Extracts top-level type, type alias, and function definitions (including
+# `model function`) from a .tppl source file, together with the comment
+# block immediately preceding each one, if any.
+#
+# Emits one record per line to stdout: name US kind US signature US comment
+# where US is \x01 and internal newlines within signature/comment are \x02,
+# so each record stays on a single physical output line (sortable, and easy
+# to split back apart by the caller).
+
+BEGIN {
+    US = sprintf("%c", 1)
+    NLc = sprintf("%c", 2)
+    OFS = US
+    depth = 0
+    capturing = 0
+    mode = ""
+    haveComment = 0
+    commentBuf = ""
+    inBlock = 0
+}
+
+function resetComment() {
+    commentBuf = ""
+    haveComment = 0
+}
+
+function addCommentLine(line,    cleaned) {
+    cleaned = line
+    sub(/^[ \t]*\/\*/, "", cleaned)
+    sub(/^[ \t]*\/\//, "", cleaned)
+    sub(/\*\/[ \t]*$/, "", cleaned)
+    sub(/^[ \t]*\*[ \t]?/, "", cleaned)
+    gsub(/^[ \t]+/, "", cleaned)
+    gsub(/[ \t]+$/, "", cleaned)
+    if (cleaned == "") return
+    if (haveComment) commentBuf = commentBuf NLc cleaned
+    else { commentBuf = cleaned; haveComment = 1 }
+}
+
+function braceDelta(line,    i, c, d) {
+    d = 0
+    for (i = 1; i <= length(line); i++) {
+        c = substr(line, i, 1)
+        if (c == "{") d++
+        else if (c == "}") d--
+    }
+    return d
+}
+
+function emit() {
+    print name, kind, sigBuf, (haveComment ? commentBuf : "")
+    resetComment()
+    capturing = 0
+    mode = ""
+    sigBuf = ""
+}
+
+# Consumes one line of a function's header (the part before its body's
+# opening brace). Once the brace is found, only the text before it is kept
+# in sigBuf, and capturing switches to silently tracking depth through the
+# body via "func-body-skip".
+function headerLine(line,    pos, prefix) {
+    pos = index(line, "{")
+    if (pos == 0) {
+        sigBuf = (sigBuf == "" ? line : sigBuf NLc line)
+        return
+    }
+    prefix = substr(line, 1, pos - 1)
+    gsub(/[ \t]+$/, "", prefix)
+    if (prefix != "") sigBuf = (sigBuf == "" ? prefix : sigBuf NLc prefix)
+    depth = braceDelta(line)
+    mode = "func-body-skip"
+    if (depth <= 0) emit()
+}
+
+{
+    line = $0
+
+    if (capturing) {
+        if (mode == "variant") {
+            t = line
+            gsub(/^[ \t]+/, "", t)
+            gsub(/[ \t]+$/, "", t)
+            if (substr(t, 1, 1) == "|") {
+                sigBuf = sigBuf NLc line
+                next
+            } else {
+                emit()
+                # fall through: reprocess this line as top-level below
+            }
+        } else if (mode == "func-header") {
+            headerLine(line)
+            next
+        } else if (mode == "func-body-skip") {
+            depth += braceDelta(line)
+            if (depth <= 0) emit()
+            next
+        } else {
+            sigBuf = sigBuf NLc line
+            depth += braceDelta(line)
+            if (depth <= 0) emit()
+            next
+        }
+    }
+
+    t = line
+    gsub(/^[ \t]+/, "", t)
+    gsub(/[ \t]+$/, "", t)
+
+    if (inBlock) {
+        addCommentLine(line)
+        if (t ~ /\*\/[ \t]*$/) inBlock = 0
+        next
+    }
+
+    if (t == "") { resetComment(); next }
+
+    if (t ~ /^\/\//) { addCommentLine(line); next }
+
+    if (t ~ /^\/\*/) {
+        addCommentLine(line)
+        if (t !~ /\*\/[ \t]*$/) inBlock = 1
+        next
+    }
+
+    if (match(t, /^(model[ \t]+)?type[ \t]+alias[ \t]+[A-Za-z_][A-Za-z0-9_]*/)) {
+        kind = "type alias"
+        name = t
+        sub(/^(model[ \t]+)?type[ \t]+alias[ \t]+/, "", name)
+        sub(/[^A-Za-z0-9_].*$/, "", name)
+    } else if (match(t, /^(model[ \t]+)?type[ \t]+[A-Za-z_][A-Za-z0-9_]*/)) {
+        kind = "type"
+        name = t
+        sub(/^(model[ \t]+)?type[ \t]+/, "", name)
+        sub(/[^A-Za-z0-9_].*$/, "", name)
+    } else if (match(t, /^(model[ \t]+)?function[ \t]+[A-Za-z_][A-Za-z0-9_]*/)) {
+        kind = "function"
+        name = t
+        sub(/^(model[ \t]+)?function[ \t]+/, "", name)
+        sub(/[^A-Za-z0-9_].*$/, "", name)
+    } else {
+        resetComment()
+        next
+    }
+
+    sigBuf = ""
+    if (kind == "type" && braceDelta(line) == 0 && t ~ /=[ \t]*$/) {
+        sigBuf = line
+        mode = "variant"
+        capturing = 1
+        next
+    }
+    if (kind == "function") {
+        mode = "func-header"
+        capturing = 1
+        headerLine(line)
+        next
+    }
+
+    sigBuf = line
+    depth = braceDelta(line)
+    if (depth <= 0) {
+        emit()
+    } else {
+        mode = "brace"
+        capturing = 1
+    }
+}

scripts/repo-utils.fish

@@ -0,0 +1,17 @@
+# Shared helpers for downloading external repositories and keeping them
+# up to date. Repositories are cloned into other-repositories/<name> at
+# the project root (which is git-ignored), and left in place between runs
+# so subsequent calls only need to pull.
+
+set -g repoDir (path resolve (status dirname)/..)
+
+set -g online yes
+
+function ensureRepo --argument-names repoName url
+    set -l dir "$repoDir/other-repositories/$repoName"
+    if not test -d $dir
+        git clone $url $dir || exit 1
+    end
+    cd $dir
+    set -qg online && begin; git pull --force || exit 1; end
+end