diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 38f5cc9d310..7cd610604e6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,12 +76,15 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} cache: npm + cache-dependency-path: tools/package-lock.json - name: Install Node.js dependencies run: npm ci + working-directory: tools - name: Test Algolia indexing run: npm run test:algolia + working-directory: tools - name: Verify Hugo modules run: hugo mod verify @@ -91,6 +94,7 @@ jobs: - name: Validate Algolia records run: npm run algolia:index -- --dry-run + working-directory: tools - name: Install XML validation tools run: sudo apt-get update && sudo apt-get install --yes libxml2-utils diff --git a/.github/workflows/update-algolia.yml b/.github/workflows/update-algolia.yml index 2521d09ff04..e9d5253db71 100644 --- a/.github/workflows/update-algolia.yml +++ b/.github/workflows/update-algolia.yml @@ -1,39 +1,79 @@ name: Update the Algolia search index + on: workflow_dispatch: schedule: - cron: "0 2 * * *" +permissions: + contents: read + +concurrency: + group: update-algolia + cancel-in-progress: false + +env: + GO_VERSION: "1.26.3" + HUGO_VERSION: "0.163.3" + NODE_VERSION: "24.18.0" + ALGOLIA_APP_ID: "LIT6P0EW26" + ALGOLIA_INDEX_NAME: "hugo" + jobs: - build: + index: + name: Build and publish index runs-on: ubuntu-latest + timeout-minutes: 30 + steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - submodules: true + fetch-depth: 0 + submodules: false lfs: false - path: website - - uses: ruby/setup-ruby@v1 + + - name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: go.sum + + - name: Set up Hugo + uses: peaceiris/actions-hugo@2752ce1d29631191ea3f27c23495fa06139a5b78 # v3 with: - bundler-cache: true - working-directory: website - - name: Check recent commit - id: checkcommit - working-directory: website - run: | - LATEST_COMMIT=$(git log -1 --format=%ct) - CURRENT_TIME=$(date +%s) - TIME_DIFF=$((CURRENT_TIME - LATEST_COMMIT)) - if [ $TIME_DIFF -gt 86400 ]; then - echo "No commit within 24 hours." - echo "recent_commit=false" >> "$GITHUB_OUTPUT" - else - echo "recent_commit=true" >> "$GITHUB_OUTPUT" - fi - - name: Build website and update Algolia index - if: ${{ github.event_name == 'workflow_dispatch' || steps.checkcommit.outputs.recent_commit == 'true' }} - working-directory: website + hugo-version: ${{ env.HUGO_VERSION }} + extended: true + + - name: Set up Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: tools/package-lock.json + + - name: Install Node.js dependencies + run: npm ci + working-directory: tools + + - name: Test Algolia indexing + run: npm run test:algolia + working-directory: tools + + - name: Verify Hugo modules + run: hugo mod verify + + - name: Build Hugo Algolia export + run: hugo --gc --minify --cleanDestinationDir --environment production + + - name: Validate generated Algolia records + run: npm run algolia:index -- --dry-run + working-directory: tools + + - name: Replace Algolia records env: - JEKYLL_ENV: production - run: ALGOLIA_API_KEY=${{ secrets.ALGOLIA_API_KEY }} bundle exec jekyll algolia + ALGOLIA_APP_ID: ${{ env.ALGOLIA_APP_ID }} + ALGOLIA_WRITE_API_KEY: ${{ secrets.ALGOLIA_API_KEY }} + ALGOLIA_INDEX_NAME: ${{ env.ALGOLIA_INDEX_NAME }} + run: npm run algolia:index + working-directory: tools diff --git a/config/_default/hugo.toml b/config/_default/hugo.toml index e912bc4055d..2c5afcfcbca 100644 --- a/config/_default/hugo.toml +++ b/config/_default/hugo.toml @@ -26,7 +26,7 @@ precice_v2_citations_url = "https://scholar.google.com/scholar?oi=bibs&hl=en&cit [params.algolia] application_id = "LIT6P0EW26" -index_name = "jekyll" +index_name = "hugo" search_only_api_key = "760ed6be3e165d08b4a798ac4aa82fe4" nodes_to_index = "p,code,table" max_record_size = 20000 diff --git a/layouts/index.algolia.json b/layouts/index.algolia.json new file mode 100644 index 00000000000..a292d4930e3 --- /dev/null +++ b/layouts/index.algolia.json @@ -0,0 +1,31 @@ +{{- $pages := slice -}} +{{- range site.Pages -}} + {{- if and (or (eq .Kind "page") (eq .Kind "section")) (ne (.Param "search") "exclude") -}} + {{- $pages = $pages | append . -}} + {{- end -}} +{{- end -}} +{{- $pages = sort $pages "RelPermalink" -}} +{ + "schemaVersion": 1, + "nodesToIndex": {{ site.Params.algolia.nodes_to_index | jsonify }}, + "maxRecordSize": {{ site.Params.algolia.max_record_size }}, + "pages": [ + {{- range $index, $page := $pages }} + {{- if $index }},{{ end }} + {{- /* Keep the search export on the same rendered-content path as page layouts. */ -}} + {{- $content := partial "compat_content.html" $page -}} + { + "title": {{ $page.Title | jsonify }}, + "url": {{ $page.RelPermalink | jsonify }}, + "html": {{ $content | jsonify }}, + "plain": {{ $content | plainify | jsonify }}, + "section": {{ $page.Section | jsonify }}, + "kind": {{ $page.Kind | jsonify }}, + "tags": {{ $page.Params.tags | jsonify }}, + "categories": {{ $page.Params.categories | jsonify }}, + "keywords": {{ $page.Params.keywords | jsonify }}, + "date": {{ if ne ($page.Lastmod.Format "2006") "0001" }}{{ $page.Lastmod.Unix }}{{ else }}null{{ end }} + } + {{- end }} + ] +} diff --git a/test/algolia-index.test.mjs b/test/algolia-index.test.mjs new file mode 100644 index 00000000000..1924b45e4f5 --- /dev/null +++ b/test/algolia-index.test.mjs @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + createRecords, + parseCliArgs, + recordByteSize, + validateSearchExport, +} from "../tools/algolia-index.mjs"; + +const searchExport = { + schemaVersion: 1, + nodesToIndex: "p,code,table", + maxRecordSize: 20_000, + pages: [ + { + title: "Example guide", + url: "/docs/example/", + html: ` +
Use precice-config to configure the adapter.
| Version | 3.4 |
/); + assert.equal(records[1].content, "precice-config"); + assert.equal(records[2].content, "Version3.4"); + assert.equal(records[0].custom_ranking.heading, 80); + assert.match(records[0].objectID, /^[a-f0-9]{64}$/); + assert.equal(records.some((record) => record.content.includes("ignored content")), false); +}); + +test("creates deterministic object IDs", () => { + const first = createRecords(searchExport).map((record) => record.objectID); + const second = createRecords(searchExport).map((record) => record.objectID); + + assert.deepEqual(first, second); +}); + +test("excludes callouts so a tutorial result starts with its introduction", () => { + const calloutExport = structuredClone(searchExport); + calloutExport.nodesToIndex = "p"; + calloutExport.pages[0].html = ` +
Note: Download the case files.
This tutorial introduces the partitioned heat-conduction case.
+ `; + calloutExport.pages[0].plain = "Note: Download the case files. This tutorial introduces the partitioned heat-conduction case."; + + const records = createRecords(calloutExport); + + assert.equal(records.length, 1); + assert.equal(records[0].content, "This tutorial introduces the partitioned heat-conduction case."); + assert.equal(records[0].content.includes("Download the case files"), false); +}); + +test("splits oversized records without exceeding the configured limit", () => { + const longExport = structuredClone(searchExport); + longExport.nodesToIndex = "p"; + longExport.pages[0].html = `${"word ".repeat(400)}
`; + longExport.pages[0].plain = "word ".repeat(400); + + const records = createRecords(longExport, { maxRecordSize: 500 }); + + assert.ok(records.length > 1); + assert.ok(records.every((record) => recordByteSize(record) <= 500)); +}); + +test("rejects invalid Hugo exports and command-line arguments", () => { + assert.throws(() => validateSearchExport({ schemaVersion: 2 }), /Unsupported Algolia export schema/); + assert.throws(() => parseCliArgs(["--batch-size", "0"]), /positive integer/); + assert.throws(() => parseCliArgs(["--unexpected"]), /Unknown option/); +}); diff --git a/tools/algolia-index.mjs b/tools/algolia-index.mjs new file mode 100644 index 00000000000..3f8a52271c6 --- /dev/null +++ b/tools/algolia-index.mjs @@ -0,0 +1,510 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { load } from "cheerio"; + +const HEADING_SELECTOR = "h1,h2,h3,h4,h5,h6"; +const EXCLUDED_SELECTOR = "script,style,iframe"; +const CALLOUT_SELECTOR = ".alert,[role='alert']"; + +// Preserve the production Algolia ranking and highlighting contract. +export const INDEX_SETTINGS = Object.freeze({ + searchableAttributes: [ + "title", + "headings", + "unordered(content)", + "collection,categories,tags", + ], + customRanking: [ + "desc(date)", + "desc(custom_ranking.heading)", + "asc(custom_ranking.position)", + ], + unretrievableAttributes: ["custom_ranking"], + attributesToHighlight: [ + "title", + "headings", + "content", + "html", + "collection", + "categories", + "tags", + ], + highlightPreTag: '', + highlightPostTag: "", + attributesToSnippet: ["content:55"], + snippetEllipsisText: "…", + distinct: true, + attributeForDistinct: "url", + attributesForFaceting: [ + "type", + "searchable(collection)", + "searchable(categories)", + "searchable(tags)", + "searchable(title)", + ], +}); + +export function parseCliArgs(args) { + const options = { + batchSize: 1000, + dryRun: false, + input: "public/algolia.json", + maxRecordSize: undefined, + }; + + for (let index = 0; index < args.length; index += 1) { + const argument = args[index]; + + if (argument === "--dry-run") { + options.dryRun = true; + continue; + } + + if (argument === "--help" || argument === "-h") { + options.help = true; + continue; + } + + if (argument === "--input" || argument === "--batch-size" || argument === "--max-record-size") { + const value = args[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`Missing value for ${argument}.`); + } + index += 1; + + if (argument === "--input") { + options.input = value; + } else if (argument === "--batch-size") { + options.batchSize = positiveInteger(value, argument); + } else { + options.maxRecordSize = positiveInteger(value, argument); + } + continue; + } + + throw new Error(`Unknown option: ${argument}`); + } + + return options; +} + +export function validateSearchExport(searchExport) { + if (!searchExport || typeof searchExport !== "object" || Array.isArray(searchExport)) { + throw new Error("Algolia export must be a JSON object."); + } + if (searchExport.schemaVersion !== 1) { + throw new Error(`Unsupported Algolia export schema: ${searchExport.schemaVersion}.`); + } + if (!Array.isArray(searchExport.pages)) { + throw new Error("Algolia export must contain a pages array."); + } + if (typeof searchExport.nodesToIndex !== "string" || !searchExport.nodesToIndex.trim()) { + throw new Error("Algolia export must define nodesToIndex."); + } + if (!Number.isInteger(searchExport.maxRecordSize) || searchExport.maxRecordSize <= 0) { + throw new Error("Algolia export must define a positive integer maxRecordSize."); + } + + for (const page of searchExport.pages) { + if (!page || typeof page !== "object") { + throw new Error("Algolia export contains an invalid page."); + } + if (typeof page.title !== "string" || typeof page.url !== "string") { + throw new Error("Every Algolia export page must contain title and url strings."); + } + if (typeof page.html !== "string" || typeof page.plain !== "string") { + throw new Error("Every Algolia export page must contain html and plain strings."); + } + } + + return searchExport; +} + +export function createRecords(searchExport, options = {}) { + validateSearchExport(searchExport); + + const maxRecordSize = options.maxRecordSize ?? searchExport.maxRecordSize; + if (!Number.isInteger(maxRecordSize) || maxRecordSize <= 0) { + throw new Error("maxRecordSize must be a positive integer."); + } + + const records = []; + for (const page of searchExport.pages) { + const pageRecords = extractPageRecords(page, searchExport.nodesToIndex); + for (const record of pageRecords) { + records.push(...fitRecord(record, maxRecordSize)); + } + } + + if (records.length === 0) { + throw new Error("The Algolia export did not produce any indexable records."); + } + + return records; +} + +export function recordByteSize(record) { + return Buffer.byteLength(JSON.stringify(record), "utf8"); +} + +export async function readSearchExport(inputPath) { + let source; + try { + source = await readFile(inputPath, "utf8"); + } catch (error) { + throw new Error(`Unable to read Algolia export at ${inputPath}: ${error.message}`); + } + + try { + return validateSearchExport(JSON.parse(source)); + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Algolia export at ${inputPath} is not valid JSON: ${error.message}`); + } + throw error; + } +} + +export async function uploadRecords({ appId, apiKey, batchSize, indexName, records }) { + const { algoliasearch } = await import("algoliasearch"); + const client = algoliasearch(appId, apiKey); + + const settingsResponse = await client.setSettings({ + indexName, + indexSettings: INDEX_SETTINGS, + }); + await client.waitForTask({ indexName, taskID: settingsResponse.taskID }); + + return client.replaceAllObjects({ + indexName, + objects: records, + batchSize, + scopes: ["settings", "rules", "synonyms"], + }); +} + +export async function run(options, environment = process.env, logger = console) { + const inputPath = path.resolve(options.input); + const searchExport = await readSearchExport(inputPath); + const records = createRecords(searchExport, options); + const maxRecordSize = options.maxRecordSize ?? searchExport.maxRecordSize; + + logger.log( + `Prepared ${records.length} Algolia records from ${searchExport.pages.length} pages ` + + `(maximum record size: ${maxRecordSize} bytes).`, + ); + + if (options.dryRun) { + logger.log("Dry run complete. No Algolia request was made."); + return records; + } + + const appId = environment.ALGOLIA_APP_ID; + const apiKey = environment.ALGOLIA_WRITE_API_KEY; + const indexName = environment.ALGOLIA_INDEX_NAME; + const missing = [ + ["ALGOLIA_APP_ID", appId], + ["ALGOLIA_WRITE_API_KEY", apiKey], + ["ALGOLIA_INDEX_NAME", indexName], + ] + .filter(([, value]) => !value) + .map(([name]) => name); + + if (missing.length > 0) { + throw new Error(`Missing required environment variables: ${missing.join(", ")}.`); + } + + await uploadRecords({ + appId, + apiKey, + batchSize: options.batchSize, + indexName, + records, + }); + logger.log(`Replaced all records in Algolia index ${indexName}.`); + + return records; +} + +function extractPageRecords(page, nodesToIndex) { + const $ = load(page.html); + let nodes; + try { + nodes = $(`${HEADING_SELECTOR},${nodesToIndex}`).toArray(); + } catch (error) { + throw new Error(`Invalid nodesToIndex selector ${JSON.stringify(nodesToIndex)}: ${error.message}`); + } + + const hierarchy = Array(6).fill(null); + const shared = sharedPageFields(page); + const records = []; + let anchor = ""; + let headingLevel = null; + let position = 0; + + for (const node of nodes) { + const element = $(node); + if (isExcludedFromSearch(element)) { + continue; + } + + const tagName = node.tagName?.toLowerCase(); + + if (tagName && /^h[1-6]$/.test(tagName)) { + headingLevel = Number(tagName[1]) - 1; + hierarchy[headingLevel] = normaliseWhitespace(element.text()); + hierarchy.fill(null, headingLevel + 1); + anchor = findAnchor($, element); + } + + if (!element.is(nodesToIndex)) { + continue; + } + + const searchableElement = element.clone(); + searchableElement.find(`${EXCLUDED_SELECTOR},${CALLOUT_SELECTOR}`).remove(); + const content = normaliseWhitespace(searchableElement.text()); + if (!content) { + continue; + } + + records.push({ + ...shared, + anchor, + content, + custom_ranking: { + heading: headingWeight(headingLevel), + position, + }, + headings: hierarchy.filter(Boolean), + html: $.html(searchableElement).trim(), + }); + position += 1; + } + + if (records.length > 0) { + return records; + } + + const fallbackContent = searchableDocumentText($) || page.title; + return [ + { + ...shared, + anchor: "", + content: fallbackContent, + custom_ranking: { heading: 100, position: 0 }, + headings: [], + html: escapeHtml(fallbackContent), + }, + ]; +} + +function isExcludedFromSearch(element) { + return element.is(EXCLUDED_SELECTOR) || element.closest(`${EXCLUDED_SELECTOR},${CALLOUT_SELECTOR}`).length > 0; +} + +function searchableDocumentText($) { + const document = $.root().clone(); + document.find(`${EXCLUDED_SELECTOR},${CALLOUT_SELECTOR}`).remove(); + return normaliseWhitespace(document.text()); +} + +function sharedPageFields(page) { + const record = { + categories: normaliseArray(page.categories), + collection: page.section || undefined, + date: Number.isInteger(page.date) ? page.date : undefined, + keywords: normaliseSearchValue(page.keywords), + tags: normaliseArray(page.tags), + title: page.title, + type: page.kind || "page", + url: page.url, + }; + + return removeUndefined(record); +} + +function fitRecord(record, maxRecordSize) { + const completed = withObjectId(record); + if (recordByteSize(completed) <= maxRecordSize) { + return [completed]; + } + + const baseRecord = { ...record }; + const fragments = []; + const words = record.content.match(/\S+/gu) ?? []; + let fragment = ""; + + for (const word of words) { + const candidate = fragment ? `${fragment} ${word}` : word; + if (fitsAsTextRecord(baseRecord, candidate, fragments.length, maxRecordSize)) { + fragment = candidate; + continue; + } + + if (fragment) { + fragments.push(textRecord(baseRecord, fragment, fragments.length)); + fragment = ""; + } + + if (fitsAsTextRecord(baseRecord, word, fragments.length, maxRecordSize)) { + fragment = word; + continue; + } + + fragments.push(...splitLongWord(baseRecord, word, fragments.length, maxRecordSize)); + } + + if (fragment) { + fragments.push(textRecord(baseRecord, fragment, fragments.length)); + } + + if (fragments.length === 0) { + throw new Error(`Record for ${record.url} cannot fit within ${maxRecordSize} bytes.`); + } + + return fragments; +} + +function fitsAsTextRecord(baseRecord, content, fragmentIndex, maxRecordSize) { + return recordByteSize(textRecord(baseRecord, content, fragmentIndex)) <= maxRecordSize; +} + +function splitLongWord(baseRecord, word, startIndex, maxRecordSize) { + const fragments = []; + let fragment = ""; + + for (const character of word) { + const candidate = `${fragment}${character}`; + if (fitsAsTextRecord(baseRecord, candidate, startIndex + fragments.length, maxRecordSize)) { + fragment = candidate; + continue; + } + + if (!fragment) { + throw new Error(`Record for ${baseRecord.url} cannot fit within ${maxRecordSize} bytes.`); + } + + fragments.push(textRecord(baseRecord, fragment, startIndex + fragments.length)); + fragment = character; + } + + if (fragment) { + fragments.push(textRecord(baseRecord, fragment, startIndex + fragments.length)); + } + + return fragments; +} + +function textRecord(baseRecord, content, fragmentIndex) { + return withObjectId({ + ...baseRecord, + content, + html: escapeHtml(content), + custom_ranking: { + ...baseRecord.custom_ranking, + position: baseRecord.custom_ranking.position + fragmentIndex / 1000, + }, + }); +} + +function withObjectId(record) { + const objectID = createHash("sha256") + .update(JSON.stringify(record)) + .digest("hex"); + return { ...record, objectID }; +} + +function findAnchor($, element) { + const directAnchor = element.attr("name") || element.attr("id"); + if (directAnchor) { + return directAnchor; + } + + const descendant = element.find("[name],[id]").first(); + return descendant.attr("name") || descendant.attr("id") || ""; +} + +function headingWeight(level) { + return level === null ? 100 : 100 - (level + 1) * 10; +} + +function normaliseWhitespace(value) { + return String(value ?? "").replace(/\s+/gu, " ").trim(); +} + +function normaliseArray(value) { + if (value === null || value === undefined || value === "") { + return []; + } + return Array.isArray(value) ? value.map(String).filter(Boolean) : [String(value)]; +} + +function normaliseSearchValue(value) { + if (value === null || value === undefined || value === "") { + return undefined; + } + return Array.isArray(value) ? value.map(String).filter(Boolean) : String(value); +} + +function removeUndefined(object) { + return Object.fromEntries(Object.entries(object).filter(([, value]) => value !== undefined)); +} + +function escapeHtml(value) { + return value.replace(/[&<>"']/gu, (character) => { + const replacements = { + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", + }; + return replacements[character]; + }); +} + +function positiveInteger(value, optionName) { + const number = Number(value); + if (!Number.isInteger(number) || number <= 0) { + throw new Error(`${optionName} must be a positive integer.`); + } + return number; +} + +function printHelp() { + console.log(`Usage: npm run algolia:index -- [options] + +Options: + --input