Implement MrDocs + Antora + mkDocs based documentation integration and website reimplementation

.devcontainer/devcontainer.json

@@ -27,10 +27,10 @@
       ]
     }
   },
-  "forwardPorts": [3000],
+  "forwardPorts": [8000],
   "containerUser": "vscode",
-  "postCreateCommand": "yarn install && curl -fsSL https://get.trunk.io -o install.sh && chmod +x install.sh && ./install.sh",
-  "waitFor": "postCreateCommand", // otherwise automated jest tests fail
+  "postCreateCommand": "sudo apt-get update && sudo apt-get install -y doxygen graphviz pandoc && npm ci && curl -fsSL https://get.trunk.io -o install.sh && chmod +x install.sh && ./install.sh",
+  "waitFor": "postCreateCommand",
   "features": {
     "node": {
       "version": "22"

.github/workflows/deploy.yml

@@ -6,49 +6,83 @@ on:
   push:
     branches:
       - main
+  schedule:
+    - cron: 0 */6 * * *
+  workflow_dispatch:
+
+concurrency:
+  group: gh-pages-publish
+  cancel-in-progress: false
 
 permissions:
-  contents: read
+  contents: write
 
 jobs:
-  build:
-    name: Build static content
+  publish:
     runs-on: ubuntu-latest
+
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout website repository
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: yarn
 
-      - name: Install dependencies
-        run: yarn install --frozen-lockfile
-      - name: Build website
-        run: yarn build
-        env:
-          PAGES_DEPLOY: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+          cache: pip
 
-      - name: Upload Build Artifact
-        uses: actions/upload-pages-artifact@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
         with:
-          path: build
+          node-version: "20"
+          cache: npm
 
-  deploy:
-    name: Deploy to GitHub Pages
-    needs: build
+      - name: Install docs toolchain
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y doxygen graphviz pandoc
+          pandoc --version
 
-    permissions:
-      pages: write
-      id-token: write
+      - name: Install MrDocs
+        run: |
+          gh release download v0.8.0 \
+            --repo cppalliance/mrdocs \
+            --pattern "MrDocs-0.8.0-Linux.tar.gz" \
+            --dir "${RUNNER_TEMP}"
+          tar -xzf "${RUNNER_TEMP}/MrDocs-0.8.0-Linux.tar.gz" -C "${RUNNER_TEMP}"
+          echo "${RUNNER_TEMP}/MrDocs-0.8.0-Linux/bin" >> "${GITHUB_PATH}"
+          "${RUNNER_TEMP}/MrDocs-0.8.0-Linux/bin/mrdocs" --version
+        env:
+          GH_TOKEN: ${{ github.token }}
 
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
+      - name: Install website dependencies
+        run: make install
 
-    runs-on: ubuntu-latest
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
+      - name: Build staged website
+        run: |
+          if [ "${GITHUB_REPOSITORY}" = "bemanproject/website" ]; then
+            export BEMAN_SITE_URL="https://bemanproject.org"
+            export BEMAN_BASE_URL="/"
+          else
+            export BEMAN_SITE_URL="https://${GITHUB_REPOSITORY_OWNER}.github.io"
+            export BEMAN_BASE_URL="/${GITHUB_REPOSITORY#*/}/"
+          fi
+          export BEMAN_GITHUB_ORG="${GITHUB_REPOSITORY_OWNER}"
+          export BEMAN_GITHUB_REPO="${GITHUB_REPOSITORY#*/}"
+          python3 scripts/run-staged-website.py build \
+            --work-root "${RUNNER_TEMP}/beman-website-work" \
+            --repos-root "${RUNNER_TEMP}/beman-external" \
+            --clone-missing \
+            --update-repos
+        env:
+          PAGES_DEPLOY: true
+
+      - name: Configure git identity
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Publish to gh-pages branch
+        run: bash scripts/publish-gh-pages.sh gh-pages

.trunk/trunk.yaml

@@ -18,6 +18,8 @@ runtimes:
 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
 lint:
   ignore:
+    - linters: [prettier]
+      paths: [mkdocs_theme/main.html]
     - linters: [osv-scanner]
       paths: [yarn.lock]
     - linters: [ALL]

CONTRIBUTING.md

@@ -2,16 +2,44 @@
 
 ## Install dependencies
 
-In order to setup the project, you have to install `npm`, `yarn` and `node`.
+In order to set up the project, use Python and Node.js/npm and install the
+dependencies from `requirements.txt` and `package-lock.json`. If you use
+micromamba, activate the project environment first:
+
+```shell
+$ micromamba activate beman
+```
 
 ```shell
 $ make install
 ```
 
-You can verify `yarn` is properly installed using
+`make install` runs `python -m pip install -r requirements.txt` and `npm ci`.
+With the `beman` micromamba environment active, the Python packages are installed
+into that environment. The npm packages are installed into this repo's
+`node_modules`.
+
+After installation, Make and npm invoke the same composed website pipeline:
+
+| Operation                         | Make         | npm             |
+| --------------------------------- | ------------ | --------------- |
+| Build and start a local server    | `make start` | `npm start`     |
+| Build static output               | `make build` | `npm run build` |
+| Build and serve the static output | `make serve` | `npm run serve` |
+
+The staged site also builds Antora documentation. Pandoc converts Markdown into
+AsciiDoc, and MrDocs generates API reference pages. `make install` does not
+install `pandoc` or `mrdocs`; make sure both are already available on `PATH`
+before running `make start` or `make build`.
+
+Library docs are configured in `beman_libraries_to_import.yaml`. Keep library
+repos adjacent to this repo (`../optional`, `../execution`, ...). Add or edit a
+library there, then run `make start`.
+
+You can verify MkDocs is properly installed using
 
 ```shell
-yarn --version
+$ python -m mkdocs --version
 ```
 
 If this fails, please check manual instructions:
@@ -33,9 +61,8 @@ This project includes a development container configuration for VS Code. To use
 <summary> Linux instructions</summary>
 
 ```shell
-$ sudo apt install nodejs
-$ sudo apt install npm
-$ npm install -g yarn
+$ sudo apt install python3 python3-pip
+$ python3 -m pip install -r requirements.txt
 ```
 
 </details>
@@ -44,9 +71,8 @@ $ npm install -g yarn
 <summary> MacOS instructions</summary>
 
 ```shell
-$ brew install node
-$ brew install npm
-$ npm install -g yarn
+$ brew install python
+$ python3 -m pip install -r requirements.txt
 ```
 
 </details>
@@ -55,8 +81,8 @@ $ npm install -g yarn
 <summary> Windows instructions</summary>
 
 ```shell
-$ winget install OpenJS.NodeJS
-$ npm install -g yarn
+$ winget install Python.Python.3
+$ python -m pip install -r requirements.txt
 ```
 
 </details>
@@ -67,16 +93,22 @@ To start a local development server, run:
 
 ```shell
 $ make start
+# or, after dependencies are installed:
+$ npm start
 ```
 
-If everything is properly installed, the command should open a browser window on http://localhost:3000/.
+If everything is properly installed, the command builds the composed MkDocs +
+Antora site and serves it on http://127.0.0.1:8000/.
 
-Most changes are reflected live without having to restart the server.
+Rerun `make start` after making documentation changes. The local server serves
+the static composed output; it is not a live-reloading MkDocs-only server.
 
 ## Generate static content for GitHub Pages deployment
 
 To generate static content from the project that can be served using any static contents hosting service (like `gh-pages`).
 
 ```shell
 $ make build
+# or:
+$ npm run build
 ```

Makefile

@@ -1,15 +1,21 @@
-.PHONY: all install build start trunk-check
+PYTHON ?= python3
 
-all: install start
+.PHONY: all install build start serve lint
+
+all: start
 
 install: # Install dependencies
-	yarn install
+	$(PYTHON) -m pip install -r requirements.txt
+	npm ci
 
-start: # Start local development server
-	yarn start
+start: install # Install dependencies and start local development server
+	$(PYTHON) scripts/run-staged-website.py start
 
 build: # Generate static content for GitHub Pages deployment
-	yarn build
+	$(PYTHON) scripts/run-staged-website.py build
+
+serve: # Build and serve the staged static site
+	$(PYTHON) scripts/run-staged-website.py serve
 
 lint: # Run code quality checks
 	trunk check

README.md

@@ -2,9 +2,10 @@
 
 This is the repository hosting the code for the future Beman Website: https://bemanproject.org/.
 
-This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator.
-Documentation is written in MDX format.
-Building and deploying it requires Node and NPM.
+This website is built using [MkDocs](https://www.mkdocs.org/), a static website generator.
+Documentation is written in Markdown.
+Building and deploying it requires Python, Node.js/npm, and the dependencies listed
+in `requirements.txt` and `package-lock.json`.
 
 ## Add a blog post
 
@@ -101,24 +102,7 @@ The CI preview deployment logs should be public. Please ping a codeowner otherwi
 <details>
 <summary> [DEBUG] Inspect local deployment error logs. </summary>
 
-- On local setup, run `make` (see [CONTRIBUTING.md](CONTRIBUTING.md#development)) and check if there is any error in the console - example:
-
-```shell
-$ make
-...
-[INFO] Starting the development server...
-...
-[ERROR] Error: Processing of blog source file path=2000-10-30-my-blog-example/index.md failed.
-    at doProcessBlogSourceFile (/Users/dariusn/dev/dn/git/Beman/website/node_modules/@docusaurus/plugin-content-blog/lib/blogUtils.js:268:19)
-    at async Promise.all (index 0)
-    ... 10 lines matching cause stack trace ...
-    at async file:///Users/dariusn/dev/dn/git/Beman/website/node_modules/@docusaurus/core/bin/docusaurus.mjs:44:3 {
-  [cause]: Error: Blog author with key "neatudarius" not found in the authors map file.
-  Valid author keys are:
-  - JeffGarland
-  - dabrahams
-  - DavidSankel
-```
+- On local setup, run `make` (see [CONTRIBUTING.md](CONTRIBUTING.md#development)) and check if there is any error in the console.
 
 - Fix the error, re-deploy the local website.
 
@@ -137,3 +121,63 @@ $ make
 ## Development
 
 Local setup, dependencies, and running the site: see **[CONTRIBUTING.md](CONTRIBUTING.md#development)**.
+
+`make` and `make start` are equivalent: both install dependencies, build the
+composed MkDocs + Antora site into a staging directory, and serve that static
+output locally. MkDocs owns the homepage, library index, talks, and blog.
+Antora owns `/docs`, including converted library READMEs, extra Markdown pages,
+and MrDocs API reference pages.
+
+### Antora documentation
+
+The staged website includes Antora-generated documentation under `/docs`.
+Pandoc converts configured Markdown pages to AsciiDoc. MrDocs emits API
+reference AsciiDoc, and Antora Collector imports those pages into each library
+component.
+
+The library list and extra Markdown pages come from
+`beman_libraries_to_import.yaml`. Keep library repos adjacent to this website
+repo, e.g. `../optional`, `../execution`, `../task`. Top-level library
+`README.md` files are included automatically on each library overview page.
+
+Required tools:
+
+- Node.js/npm dependencies from `package-lock.json`
+- `pandoc` on `PATH`
+- `mrdocs` on `PATH`
+
+With the `beman` micromamba environment active:
+
+```shell
+$ make start
+$ make build
+```
+
+To add a library, edit `beman_libraries_to_import.yaml`, put the repo next to
+`website`, then run `make start`.
+
+The first Antora build downloads the default UI bundle; later builds reuse the
+local Antora cache under `build/antora-cache`.
+
+## Automated `gh-pages` publishing
+
+GitHub Actions publishes this site to the `gh-pages` branch on:
+
+- pushes to `main`
+- a 6-hour schedule (`0 */6 * * *`)
+- manual dispatch
+
+```shell
+$ python3 scripts/run-staged-website.py build --repos-root /tmp/beman-external --clone-missing --update-repos
+```
+
+For builds published from a fork or any GitHub Pages project site, set the site
+URL and base URL to match the repository path. Example:
+
+```shell
+$ BEMAN_SITE_URL="https://<your_username>.github.io" \
+  BEMAN_BASE_URL="/beman-website/" \
+  BEMAN_GITHUB_ORG="<your_username>" \
+  BEMAN_GITHUB_REPO="beman-website" \
+  python3 scripts/run-staged-website.py build --repos-root ..
+```