feat(sitemap): add accurate lastmod from each doc's last content change

.github/workflows/build_and_check.yml

@@ -21,6 +21,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          # Full history, so the preview's last-update dates and sitemap
+          # <lastmod> match what production will ship.
+          fetch-depth: 0
 
       - name: Set up Node.js
         uses: actions/setup-node@v5
@@ -39,6 +43,9 @@ jobs:
       - name: Verify structured-data graph
         run: npm run verify-schema
 
+      - name: Verify sitemap lastmod dates
+        run: npm run verify-lastmod
+
       # Deploys this PR's build with main.yml's deploy script into a local S3,
       # then replays the weeks after it: no asset a live or open page still
       # loads is deleted, every unused one eventually is, and every file gets

.github/workflows/main.yml

@@ -47,6 +47,9 @@ jobs:
         with:
           # main as it is now, not github.sha — see the concurrency comment.
           ref: main
+          # Full history: doc last-update dates and sitemap <lastmod> come
+          # from each file's git log (src/lastUpdate/contentLastUpdate.js).
+          fetch-depth: 0
 
       - name: Set Node.js 22.x
         uses: actions/setup-node@v3

.lastmod-ignore-revs

@@ -0,0 +1,11 @@
+# Commits that change doc bodies without changing what they say (a mass
+# find-and-replace, a link or formatting rewrite). Doc last-update dates and
+# the sitemap <lastmod> skip them. Front-matter-only and whitespace-only
+# changes are already skipped automatically, so they do not need listing.
+# One full commit hash per line; see src/lastUpdate/contentLastUpdate.js.
+
+# #896: swapped <img> for webp and gifs for <video>; no text changed.
+fd27e0f56a544fd29a59ec10a0abdee387e1cf46
+
+# #897: SEO sweep (Related footers, HowTo blocks, alt text, dashes, prettier).
+80307a582dec2bd7be9e5b5802cfe71655fb88b6

docusaurus.config.js

@@ -8,6 +8,7 @@ const fs = require("fs");
 const remarkImageSize = require("./src/remark/remarkImageSize");
 import {visit} from "unist-util-visit";
 const FontPreloadPlugin = require("webpack-font-preload-plugin");
+const {withContentLastUpdate} = require("./src/lastUpdate/contentLastUpdate");
 
 // Single source of truth for the "current" docs version. Update this one
 // value when cutting a new docs version -- it drives both the versioning
@@ -170,6 +171,17 @@ s.parentNode.insertBefore(b, s);})(window.lintrk);`,
   baseUrl: "/docs/",
   onBrokenLinks: "throw",
   trailingSlash: true,
+  markdown: {
+    // Date each doc by the last commit that changed its content, not the last
+    // commit that touched the file (version cuts, front-matter sweeps). Feeds
+    // the "Last updated" line, TechArticle dateModified and sitemap <lastmod>.
+    // See src/lastUpdate/contentLastUpdate.js.
+    parseFrontMatter: async (params) =>
+      withContentLastUpdate(
+        params.filePath,
+        await params.defaultParseFrontMatter(params)
+      ),
+  },
   favicon: "img/favicon.png",
   organizationName: "keploy", // Usually your GitHub org/user name.
   projectName: "docs", // Usually your repo name.
@@ -509,6 +521,10 @@ s.parentNode.insertBefore(b, s);})(window.lintrk);`,
           //cacheTime: 600 * 1000, // 600 sec - cache purge period
           changefreq: "weekly",
           priority: 0.5,
+          // <lastmod> from each doc's last content change (date only, which
+          // is all the precision Google uses). Only accurate with full git
+          // history: CI checks out with fetch-depth: 0.
+          lastmod: "date",
           filename: "sitemap.xml",
           // Differentiate docs sitemap priorities by content type so
           // search engines spend crawl budget proportional to how

package.json

@@ -9,6 +9,7 @@
     "build": "docusaurus build",
     "verify-schema": "node scripts/verify-schema-graph.js build",
     "verify-linkedin-tag": "node scripts/verify-linkedin-tag.js build",
+    "verify-lastmod": "node scripts/verify-lastmod.js build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",

scripts/verify-lastmod.js

@@ -0,0 +1,59 @@
+// Asserts the built sitemap carries real per-page <lastmod> dates. Catches the
+// failure mode where every page silently ships one date (a shallow clone, git
+// unavailable, the sitemap `lastmod` option dropped), which a green build
+// otherwise hides. See src/lastUpdate/contentLastUpdate.js.
+//
+// Usage: node scripts/verify-lastmod.js <build dir>
+const fs = require("fs");
+const path = require("path");
+
+// Thresholds. Today: ~216 URLs, ~46 distinct dates, the largest single date
+// (#918, a real sitewide rewrite) covers about a third of pages.
+const MIN_COVERAGE = 0.9; // share of URLs that must carry <lastmod>
+const MIN_DISTINCT_DATES = 10;
+const MAX_SHARE_OF_ONE_DATE = 0.5;
+
+const buildDir = process.argv[2] || "build";
+const sitemapPath = path.join(buildDir, "sitemap.xml");
+const xml = fs.readFileSync(sitemapPath, "utf8");
+
+const fail = (msg) => {
+  console.error(`verify-lastmod: ${msg} (${sitemapPath})`);
+  process.exit(1);
+};
+
+const urls = xml.match(/<url>[\s\S]*?<\/url>/g) || [];
+if (!urls.length) fail("no <url> entries");
+
+const dates = urls
+  .map((url) => url.match(/<lastmod>([^<]+)<\/lastmod>/))
+  .filter(Boolean)
+  .map((match) => match[1]);
+
+const coverage = dates.length / urls.length;
+if (coverage < MIN_COVERAGE) {
+  fail(`only ${dates.length} of ${urls.length} URLs have <lastmod>`);
+}
+
+const counts = {};
+for (const date of dates) counts[date] = (counts[date] || 0) + 1;
+const distinct = Object.keys(counts).length;
+if (distinct < MIN_DISTINCT_DATES) {
+  fail(`only ${distinct} distinct <lastmod> dates; is git history shallow?`);
+}
+
+const [topDate, topCount] = Object.entries(counts).sort(
+  (a, b) => b[1] - a[1]
+)[0];
+if (topCount / dates.length > MAX_SHARE_OF_ONE_DATE) {
+  fail(
+    `${topCount} of ${dates.length} URLs share <lastmod> ${topDate}; ` +
+      "list a sweep in .lastmod-ignore-revs, or raise " +
+      "MAX_SHARE_OF_ONE_DATE if the change is real"
+  );
+}
+
+console.log(
+  `verify-lastmod: ok (${dates.length}/${urls.length} URLs, ` +
+    `${distinct} distinct dates, largest ${topDate} x${topCount})`
+);