docs: publish the newest release at user-guide/latest and main at user-guide/dev

.github/labeler.yml

@@ -138,7 +138,7 @@ area:Delta:
           - 'native/core/src/execution/planner/delta_spark_scan.rs'
           - '.github/workflows/delta_*.yml'
           - 'dev/verify-contrib-delta-gate.sh'
-          - 'docs/source/user-guide/latest/delta.md'
+          - 'docs/source/user-guide/dev/delta.md'
 
 area:ci:
   - changed-files:

.prettierignore

@@ -19,7 +19,7 @@
 # single expression row rewrites every row in the table. That produces noisy
 # diffs and frequent merge conflicts between PRs that each add new expressions.
 # This file is almost entirely tables, so exempt it from prettier formatting.
-docs/source/user-guide/latest/expressions.md
+docs/source/user-guide/dev/expressions.md
 
 # Generated by `benchmarks/micro/run.py collect` on the machine that produced the
 # benchmark results, where prettier is not available.

contrib/delta-spark/README.md

@@ -32,7 +32,7 @@ Support is experimental and explicitly opt-in. Two things are required:
    alone does nothing.
 
 Unsupported tables and features fall back to Spark's reader. See the
-[user guide](https://datafusion.apache.org/comet/user-guide/latest/delta.html)
+[user guide](https://datafusion.apache.org/comet/user-guide/dev/delta.html)
 for configuration details.
 
 ## Supported versions

contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala

@@ -776,9 +776,9 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
     // DeltaSparkConfigProvider with the current module layout, so this is the only check tying
     // each entry's key, doc string, and default to the row GenerateDocs would render.
     val file = DeltaScanContribSuite
-      .findRepoFile("docs/source/user-guide/latest/delta.md")
+      .findRepoFile("docs/source/user-guide/dev/delta.md")
       .getOrElse(
-        fail("Could not locate docs/source/user-guide/latest/delta.md from this checkout; " +
+        fail("Could not locate docs/source/user-guide/dev/delta.md from this checkout; " +
           "set -Dcomet.repo.root or run from the repo or module root"))
     val source = scala.io.Source.fromFile(file, "UTF-8")
     val tableRows =

dev/generate-release-docs.sh

@@ -19,7 +19,7 @@
 # This script freezes generated documentation content onto a release branch.
 # It should be run once when creating a new release branch.
 #
-# On main, docs/source/user-guide/latest/ holds only template markers; CI fills them
+# On main, docs/source/user-guide/dev/ holds only template markers; CI fills them
 # at publish time via docs/build.sh. A release branch instead commits the generated
 # content so that the archived docs for the release render real tables (archived
 # versions are served from the frozen branch, not regenerated at publish).
@@ -37,7 +37,7 @@
 # Example workflow when cutting release 0.13.0 (release-branch changes go via PR):
 #   git checkout -b release-0.13-docs branch-0.13
 #   ./dev/generate-release-docs.sh
-#   git add docs/source/user-guide/latest/
+#   git add docs/source/user-guide/dev/
 #   git commit -m "Generate docs for 0.13.0 release"
 #   # open a PR targeting branch-0.13
 
@@ -48,7 +48,7 @@ PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
 
 cd "${PROJECT_ROOT}"
 
-LATEST_USER_GUIDE="${PROJECT_ROOT}/docs/source/user-guide/latest"
+LATEST_USER_GUIDE="${PROJECT_ROOT}/docs/source/user-guide/dev"
 COMPAT_EXPR_DIR="${LATEST_USER_GUIDE}/compatibility/expressions"
 TEMPLATE_DIR="${COMPAT_EXPR_DIR}/_category_template"
 
@@ -81,11 +81,11 @@ echo "Formatting generated Markdown with prettier..."
 npx prettier "${LATEST_USER_GUIDE}/**/*.md" --write
 
 echo ""
-echo "Done! Generated documentation content in docs/source/user-guide/latest/"
+echo "Done! Generated documentation content in docs/source/user-guide/dev/"
 echo "  - configs.md (config reference)"
 echo "  - compatibility/expressions/spark-*/ (per-Spark-version compatibility pages)"
 echo ""
 echo "Next steps (release-branch changes go via PR):"
-echo "  git add docs/source/user-guide/latest/"
+echo "  git add docs/source/user-guide/dev/"
 echo "  git commit -m 'Generate docs for release'"
 echo "  # push to your fork and open a PR targeting the release branch"

docs/README.md

@@ -72,6 +72,14 @@ To make changes to the docs, simply make a Pull Request with your
 proposed changes as normal. When the PR is merged the docs will be
 automatically updated.
 
+## Published URLs
+
+The user guide is published under `user-guide/`: `dev/` is the unreleased guide built from `main`,
+`latest/` is a copy of the newest release's guide, and each release also keeps its own versioned
+directory such as `1.1/`. Only the versioned URLs are permanent. Pages under `latest/` and `dev/`
+can be added, moved, or removed between releases, and we do not guarantee that links to them keep
+working. The release version lives in `generate-versions.py`.
+
 ## Release Process
 
 This documentation is hosted at <https://datafusion.apache.org/comet/>

docs/build.sh

@@ -34,22 +34,22 @@ rm -rf comet-1.0
 rm -rf comet-1.1
 python3 generate-versions.py
 
-# Generate dynamic content (configs, compatibility matrices) for latest docs
+# Generate dynamic content (configs, compatibility matrices) for the dev docs
 # This runs GenerateDocs against the temp copy, not source files. Each supported Spark
 # profile gets its own copy of the compatibility/expressions templates so that the
 # generated content can differ per Spark version (some expressions are only registered
 # through version-specific shims).
 echo "Generating dynamic documentation content..."
 cd ..
 
-LATEST_USER_GUIDE="$(pwd)/docs/temp/user-guide/latest"
-COMPAT_EXPR_DIR="${LATEST_USER_GUIDE}/compatibility/expressions"
+DEV_USER_GUIDE="$(pwd)/docs/temp/user-guide/dev"
+COMPAT_EXPR_DIR="${DEV_USER_GUIDE}/compatibility/expressions"
 TEMPLATE_DIR="${COMPAT_EXPR_DIR}/_category_template"
 
 # Spark profiles for which we publish per-version compatibility pages. Add or remove
 # entries here when adding/dropping support for a Spark version. Each entry must match
 # both a Maven profile id (-P<profile>) and a source-tree subdirectory under
-# docs/source/user-guide/latest/compatibility/expressions/.
+# docs/source/user-guide/dev/compatibility/expressions/.
 # Every run also rewrites the shared configs.md and expressions.md pages, so the last
 # entry decides their content. Keep the default build profile (spark-4.1) last.
 SPARK_PROFILES=(spark-3.4 spark-3.5 spark-4.0 spark-4.2 spark-4.1)
@@ -59,11 +59,15 @@ for profile in "${SPARK_PROFILES[@]}"; do
   mkdir -p "${COMPAT_EXPR_DIR}/${profile}"
   cp "${TEMPLATE_DIR}"/*.md "${COMPAT_EXPR_DIR}/${profile}/"
   ./mvnw -q package -P${profile} -Pgenerate-docs -DskipTests -Dmaven.test.skip=true \
-    -Dexec.arguments="${LATEST_USER_GUIDE},${profile}"
+    -Dexec.arguments="${DEV_USER_GUIDE},${profile}"
 done
 
 # Templates aren't a real doc page; remove from the build input so Sphinx doesn't see them.
 rm -rf "${TEMPLATE_DIR}"
 cd docs
 
 make SOURCEDIR=`pwd`/temp html
+
+# Serve the newest release's user guide at user-guide/latest as well. Run after the build because
+# it copies the finished HTML.
+python3 generate-versions.py --alias-latest build/html

docs/generate-versions.py

@@ -20,12 +20,28 @@
 
 # This script clones release branches such as branch-0.9 and then copies the user guide markdown
 # for inclusion in the published documentation so that we publish user guides for released versions
-# of Comet
+# of Comet.
+#
+# The published user guide URLs follow the convention used by Apache Spark and Apache Iceberg:
+#
+#   user-guide/dev/      the unreleased user guide on main
+#   user-guide/latest/   the newest release (an alias, see alias_latest below)
+#   user-guide/1.1/      each release, pinned
+#
+# Only the versioned URLs are permanent. What `latest` and `dev` show changes with every release,
+# and pages can be added, moved or removed between releases.
 
 import os
+import shutil
+import sys
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
+# The newest release, and the older releases that are still published. Promote a new release by
+# moving the current one into PREVIOUS_VERSIONS (see the release process guide).
+LATEST_RELEASED_VERSION = "1.1.0"
+PREVIOUS_VERSIONS = ["0.13.0", "0.14.0", "0.15.0", "0.16.0", "0.17.0", "1.0.0"]
+
 def get_major_minor_version(version: str):
     parts = version.split('.')
     return f"{parts[0]}.{parts[1]}"
@@ -111,16 +127,22 @@ def insert_warning_after_asf_header(root: str, warning: str):
         file.write_text("".join(new_lines), encoding="utf-8")
 
 
-def get_user_guide_dir(major_minor: str):
-    if major_minor == "0.8" or major_minor == "0.9":
-        return "docs/source/user-guide"
-    else:
-        return "docs/source/user-guide/latest"
+def get_user_guide_dir(clone_dir: str):
+    # Where a release branch keeps its user guide depends on when it was cut: under
+    # user-guide/latest/ up to the 1.1 branch, and under user-guide/dev/ after that.
+    for candidate in ("docs/source/user-guide/dev", "docs/source/user-guide/latest"):
+        if (Path(clone_dir) / candidate).is_dir():
+            return candidate
+    raise FileNotFoundError(
+        f"{clone_dir} has no docs/source/user-guide/dev or docs/source/user-guide/latest; "
+        "did the git clone fail?"
+    )
 
 def publish_released_version(version: str):
     major_minor = get_major_minor_version(version)
-    dir = get_user_guide_dir(major_minor)
-    os.system(f"git clone --depth 1 https://github.com/apache/datafusion-comet.git -b branch-{major_minor} comet-{major_minor}")
+    clone_dir = f"comet-{major_minor}"
+    os.system(f"git clone --depth 1 https://github.com/apache/datafusion-comet.git -b branch-{major_minor} {clone_dir}")
+    dir = get_user_guide_dir(clone_dir)
     os.system(f"mkdir temp/user-guide/{major_minor}")
     os.system(f"cp -rf comet-{major_minor}/{dir}/* temp/user-guide/{major_minor}")
     # Strip snapshot-only content; keep release-only content
@@ -131,12 +153,19 @@ def publish_released_version(version: str):
 
 def generate_docs(snapshot_version: str, latest_released_version: str, previous_versions: list[str]):
 
-    # Strip release-only content; keep snapshot-only content for the latest (snapshot) docs
-    strip_conditional_blocks("temp/user-guide/latest", "snapshot")
+    # user-guide/latest is the release alias that alias_latest replaces after the Sphinx build, so
+    # a page sourced from there would be built and then silently deleted.
+    if (Path(__file__).parent / "source/user-guide/latest").exists():
+        raise RuntimeError(
+            "docs/source/user-guide/latest is reserved for the release alias; use user-guide/dev"
+        )
+
+    # Strip release-only content; keep snapshot-only content for the dev docs
+    strip_conditional_blocks("temp/user-guide/dev", "snapshot")
 
     # Replace $COMET_VERSION with actual version for snapshot version
     for file_pattern in ["*.md", "*.rst"]:
-        replace_in_files(f"temp/user-guide/latest", file_pattern, "$COMET_VERSION", snapshot_version)
+        replace_in_files("temp/user-guide/dev", file_pattern, "$COMET_VERSION", snapshot_version)
 
     # Add user guide content for latest released versions
     publish_released_version(latest_released_version)
@@ -151,9 +180,31 @@ def generate_docs(snapshot_version: str, latest_released_version: str, previous_
         major_minor = get_major_minor_version(version)
         insert_warning_after_asf_header(f"temp/user-guide/{major_minor}", warning)
 
+def alias_latest(html_root: str):
+    """Serve the newest release's built user guide at user-guide/latest as well.
+
+    This copies the finished HTML rather than building a second copy of the sources, so Sphinx
+    sees each document once. It is a copy and not a symlink because the site is published by
+    rsync into the asf-site branch, and ASF's web server is not known to follow symlinks. Every
+    link in the pages is relative, so the copy works unchanged one directory over.
+    """
+    user_guide = Path(html_root) / "user-guide"
+    source = user_guide / get_major_minor_version(LATEST_RELEASED_VERSION)
+    if not source.is_dir():
+        raise FileNotFoundError(f"{source} was not built, so there is nothing to alias as latest")
+    target = user_guide / "latest"
+    if target.exists():
+        shutil.rmtree(target)
+    shutil.copytree(source, target)
+    print(f"Aliased {source} as {target}")
+
 if __name__ == "__main__":
+    # `generate-versions.py --alias-latest <html dir>` runs after the Sphinx build.
+    if len(sys.argv) == 3 and sys.argv[1] == "--alias-latest":
+        alias_latest(sys.argv[2])
+        sys.exit(0)
+    if len(sys.argv) != 1:
+        sys.exit("usage: generate-versions.py [--alias-latest <html dir>]")
     print("Generating versioned user guide docs...")
     snapshot_version = get_version_from_pom()
-    latest_released_version = "1.1.0"
-    previous_versions = ["0.13.0", "0.14.0", "0.15.0", "0.16.0", "0.17.0", "1.0.0"]
-    generate_docs(snapshot_version, latest_released_version, previous_versions)
+    generate_docs(snapshot_version, LATEST_RELEASED_VERSION, PREVIOUS_VERSIONS)