diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60ef72ae6..80e30550e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,6 +61,9 @@ jobs: - name: reconfigure private repository access run: ACCESS_TOKEN=${{ secrets.ACCESS_TOKEN }} task use:insider-https-token + - name: test + run: task test:unit + - name: check run: task check diff --git a/Taskfile.yml b/Taskfile.yml index 0c34d034f..a9007f198 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -48,7 +48,7 @@ tasks: - poetry install check: - desc: Run complete test suite + desc: Check documentation links and Markdown style deps: - check:links - check:rumdl @@ -60,6 +60,25 @@ tasks: cmds: - poetry run linkcheckMarkdown -r docs 2>&1 | grep -v "ResourceWarning" + test: + desc: Run the Python unit test suite + deps: + - test:unit + + test:unit: + desc: Run tests that need no external service + deps: + - install + cmds: + - poetry run pytest -m "not integration" + + test:integration: + desc: Run tests that need a live CO instance + deps: + - install + cmds: + - poetry run pytest -m integration + build: desc: Build the page deps: diff --git a/poetry.lock b/poetry.lock index 4f4bfaf3c..01f088d26 100644 --- a/poetry.lock +++ b/poetry.lock @@ -762,6 +762,17 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] type = ["pytest-mypy"] +[[package]] +name = "iniconfig" +version = "2.3.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.10" +files = [ + {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, + {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1394,6 +1405,21 @@ docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.1.3)", "sphinx-a test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.4)", "pytest-cov (>=6)", "pytest-mock (>=3.14)"] type = ["mypy (>=1.14.1)"] +[[package]] +name = "pluggy" +version = "1.6.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["coverage", "pytest", "pytest-benchmark"] + [[package]] name = "propcache" version = "0.3.2" @@ -1690,6 +1716,27 @@ files = [ [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pytest" +version = "9.1.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c"}, + {file = "pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313"}, +] + +[package.dependencies] +colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} +iniconfig = ">=1.0.1" +packaging = ">=22" +pluggy = ">=1.5,<2" +pygments = ">=2.7.2" + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2209,4 +2256,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "8dca9021c750c8ab63ab240ae05ccde92fb1998bcd1659ce19074d31610aefc9" +content-hash = "b29564f7e517705f8e3acab382a355450ede6affa5d79e085c0200fe8284f134" diff --git a/pyproject.toml b/pyproject.toml index ff86d0d68..654e2fc25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,12 @@ jinja2 = "^3.1.6" [tool.poetry.group.dev.dependencies] linkcheckmd = "^1.4.0" rumdl = "^0.0.194" +pytest = "^9.1.1" + +[tool.pytest.ini_options] +markers = [ + "integration: needs a live DI/CMEM instance to run", +] [build-system] requires = ["poetry-core"] diff --git a/tests/test_update_di_reference.py b/tests/test_update_di_reference.py index 330e4421a..d63f0c432 100644 --- a/tests/test_update_di_reference.py +++ b/tests/test_update_di_reference.py @@ -1,8 +1,226 @@ """Test update DI references""" -from tools.update_di_reference import get_plugin_descriptions +import pytest +from tools.update_di_reference import ( + PluginDescription, + PluginReference, + RelatedPluginReferenceError, + get_plugin_descriptions, + resolve_related_plugin_links, + create_plugin_markdown, + build_plugin_paths, + validate_related_plugin_references, +) + +@pytest.mark.integration def test_get_plugin_descriptions(): """Test get DI plugin descriptions""" descriptions = get_plugin_descriptions() - pass + assert set(descriptions.keys()) == {"customtask", "dataset", "distancemeasure", "transformer", "aggregator"} + assert len(descriptions["transformer"]) > 0 + assert len(descriptions["dataset"]) > 0 + for plugin_type, plugins in descriptions.items(): + for plugin in plugins: + assert isinstance(plugin, PluginDescription) + assert plugin.pluginType == plugin_type + titles = [p.title.lower() for p in plugins] + assert titles == sorted(titles) + + +def _make_plugin(plugin_id, plugin_type="transformer", main_category="Extract", related=None): + """Build a minimal valid PluginDescription for a test, filling in only what varies.""" + return PluginDescription( + pluginId=plugin_id, + title=plugin_id, + categories=[main_category], + description="test plugin", + properties={}, + actions={}, + required=[], + backendType="scala", + pluginType=plugin_type, + relatedPlugins=related or [], + ) + + +@pytest.mark.parametrize( + "current_path, target_path, expected", + [ + # same type, different category + ("transformer/Extract/regexExtract.md", "transformer/Replace/regexReplace.md", "../Replace/regexReplace.md"), + # different type entirely + ("transformer/Extract/regexExtract.md", "aggregator/average.md", "../../aggregator/average.md"), + # reverse: shallower page linking to a deeper one + ("aggregator/average.md", "transformer/Extract/regexExtract.md", "../transformer/Extract/regexExtract.md"), + # same directory + ("customtask/sparqlSelectOperator.md", "customtask/sparqlUpdateOperator.md", "sparqlUpdateOperator.md"), + ], +) +def test_resolve_related_plugin_links_path_shapes(current_path, target_path, expected): + ref = PluginReference(id="target") + plugin = _make_plugin("current", related=[ref]) + plugin_paths = {"current": current_path, "target": target_path} + assert resolve_related_plugin_links(plugin, current_path, plugin_paths) == [(ref, expected)] + + +def test_resolve_related_plugin_links_empty(): + plugin = _make_plugin("regexExtract", related=[]) + assert resolve_related_plugin_links(plugin, "transformer/Extract/regexExtract.md", {}) == [] + + +def test_resolve_related_plugin_links_single(): + ref = PluginReference(id="regexReplace", description="replaces text") + plugin = _make_plugin("regexExtract", related=[ref]) + plugin_paths = {"regexExtract": "transformer/Extract/regexExtract.md", "regexReplace": "transformer/Replace/regexReplace.md"} + resolved = resolve_related_plugin_links(plugin, "transformer/Extract/regexExtract.md", plugin_paths) + assert resolved == [(ref, "../Replace/regexReplace.md")] + + +def test_resolve_related_plugin_links_multiple_preserves_order(): + ref_a = PluginReference(id="regexReplace") + ref_b = PluginReference(id="regexSelect") + plugin = _make_plugin("regexExtract", related=[ref_a, ref_b]) + plugin_paths = { + "regexExtract": "transformer/Extract/regexExtract.md", + "regexReplace": "transformer/Replace/regexReplace.md", + "regexSelect": "transformer/Selection/regexSelect.md", + } + resolved = resolve_related_plugin_links(plugin, "transformer/Extract/regexExtract.md", plugin_paths) + assert [ref for ref, _ in resolved] == [ref_a, ref_b] + + +def test_resolve_related_plugin_links_unresolvable_raises(): + ref = PluginReference(id="deprecatedPlugin") + plugin = _make_plugin("regexExtract", related=[ref]) + with pytest.raises(RelatedPluginReferenceError, match="deprecatedPlugin"): + resolve_related_plugin_links(plugin, "transformer/Extract/regexExtract.md", {"regexExtract": "transformer/Extract/regexExtract.md"}) + + +def test_build_plugin_paths(): + plugins = { + "transformer": [_make_plugin("regexExtract", plugin_type="transformer", main_category="Extract")], + "dataset": [_make_plugin("sparqlEndpoint", plugin_type="dataset")], + "customtask": [_make_plugin("deprecatedPlugin", plugin_type="customtask")], + } + paths = build_plugin_paths(plugins) + assert paths == { + "regexExtract": "transformer/Extract/regexExtract.md", + "sparqlEndpoint": "dataset/sparqlEndpoint.md", + } + assert "deprecatedPlugin" not in paths + + +def test_validate_related_plugin_references_passes_when_all_resolve(): + plugins = { + "transformer": [_make_plugin("regexExtract", related=[PluginReference(id="regexReplace")])], + } + plugin_paths = { + "regexExtract": "transformer/Extract/regexExtract.md", + "regexReplace": "transformer/Replace/regexReplace.md", + } + validate_related_plugin_references(plugins, plugin_paths) + + +def test_validate_related_plugin_references_raises_on_unresolvable(): + plugins = { + "transformer": [_make_plugin("regexExtract", related=[PluginReference(id="removedPlugin")])], + } + plugin_paths = {"regexExtract": "transformer/Extract/regexExtract.md"} + with pytest.raises(RelatedPluginReferenceError, match="removedPlugin"): + validate_related_plugin_references(plugins, plugin_paths) + + +def test_validate_related_plugin_references_skips_deprecated_plugins(): + plugins = { + "customtask": [_make_plugin("deprecatedPlugin", plugin_type="customtask", related=[PluginReference(id="removedPlugin")])], + } + validate_related_plugin_references(plugins, plugin_paths={}) + + +def test_create_plugin_markdown_writes_resolved_links(tmp_path): + plugin = _make_plugin( + "sparqlSelectOperator", + plugin_type="customtask", + related=[PluginReference(id="sparqlEndpoint", description="a SPARQL endpoint dataset")], + ) + plugin_paths = { + "sparqlSelectOperator": "customtask/sparqlSelectOperator.md", + "sparqlEndpoint": "dataset/sparqlEndpoint.md", + } + + create_plugin_markdown(plugin, tmp_path, plugin_paths) + + written = (tmp_path / "customtask" / "sparqlSelectOperator.md").read_text() + assert "[sparqlEndpoint](../dataset/sparqlEndpoint.md)" in written + assert "**sparqlEndpoint**" not in written + assert "[sparqlEndpoint](../dataset/sparqlEndpoint.md) — a SPARQL endpoint dataset" in written + + +@pytest.mark.parametrize( + "plugin_id, plugin_path, related_id, related_path, expected_link", + [ + # same type, different category + ("regexExtract", "transformer/Extract/regexExtract.md", "regexReplace", "transformer/Replace/regexReplace.md", "../Replace/regexReplace.md"), + # reverse: shallower page linking to a deeper one + ("average", "aggregator/average.md", "regexExtract", "transformer/Extract/regexExtract.md", "../transformer/Extract/regexExtract.md"), + # same directory + ("sparqlSelectOperator", "customtask/sparqlSelectOperator.md", "sparqlUpdateOperator", "customtask/sparqlUpdateOperator.md", "sparqlUpdateOperator.md"), + ], +) +def test_create_plugin_markdown_link_path_shapes(tmp_path, plugin_id, plugin_path, related_id, related_path, expected_link): + plugin = _make_plugin(plugin_id, related=[PluginReference(id=related_id)]) + plugin_paths = {plugin_id: plugin_path, related_id: related_path} + + create_plugin_markdown(plugin, tmp_path, plugin_paths) + + written = (tmp_path / plugin_path).read_text() + assert f"[{related_id}]({expected_link})" in written + + +def test_create_plugin_markdown_writes_multiple_resolved_links(tmp_path): + plugin = _make_plugin( + "regexExtract", + plugin_type="transformer", + main_category="Extract", + related=[PluginReference(id="regexReplace"), PluginReference(id="regexSelect")], + ) + plugin_paths = { + "regexExtract": "transformer/Extract/regexExtract.md", + "regexReplace": "transformer/Replace/regexReplace.md", + "regexSelect": "transformer/Selection/regexSelect.md", + } + + create_plugin_markdown(plugin, tmp_path, plugin_paths) + + written = (tmp_path / "transformer" / "Extract" / "regexExtract.md").read_text() + assert "[regexReplace](../Replace/regexReplace.md)" in written + assert "[regexSelect](../Selection/regexSelect.md)" in written + + +def test_create_plugin_markdown_omits_description_when_absent(tmp_path): + plugin = _make_plugin( + "sparqlSelectOperator", + plugin_type="customtask", + related=[PluginReference(id="sparqlEndpoint")], + ) + plugin_paths = { + "sparqlSelectOperator": "customtask/sparqlSelectOperator.md", + "sparqlEndpoint": "dataset/sparqlEndpoint.md", + } + + create_plugin_markdown(plugin, tmp_path, plugin_paths) + + written = (tmp_path / "customtask" / "sparqlSelectOperator.md").read_text() + assert "[sparqlEndpoint](../dataset/sparqlEndpoint.md)" in written + assert "[sparqlEndpoint](../dataset/sparqlEndpoint.md) —" not in written + + +def test_create_plugin_markdown_omits_section_when_no_related_plugins(tmp_path): + plugin = _make_plugin("regexExtract", plugin_type="transformer", main_category="Extract", related=[]) + plugin_paths = {"regexExtract": "transformer/Extract/regexExtract.md"} + + create_plugin_markdown(plugin, tmp_path, plugin_paths) + + written = (tmp_path / "transformer" / "Extract" / "regexExtract.md").read_text() + assert "Related Plugins" not in written diff --git a/tools/templates/plugin.md b/tools/templates/plugin.md index 3f9bb736f..e22400748 100644 --- a/tools/templates/plugin.md +++ b/tools/templates/plugin.md @@ -26,11 +26,11 @@ tags: {% for tag in plugin.tags %} {{parameters_advanced if plugin.properties_advanced else "`None`"}} -{%- if plugin.relatedPlugins %} +{%- if related_plugins_resolved %} ## Related Plugins -{% for ref in plugin.relatedPlugins -%} -- **{{ ref.id }}**{% if ref.description %} — {{ ref.description }}{% endif %} +{% for ref, link in related_plugins_resolved -%} +- [{{ ref.id }}]({{ link }}){% if ref.description %} — {{ ref.description }}{% endif %} {% endfor %} {%- endif %} diff --git a/tools/update_di_reference.py b/tools/update_di_reference.py index 1f0d2c07c..6894509e0 100644 --- a/tools/update_di_reference.py +++ b/tools/update_di_reference.py @@ -1,6 +1,7 @@ """Update DI Reference documentation""" import json +import posixpath import re from contextlib import suppress from pathlib import Path @@ -161,13 +162,43 @@ def get_plugin_descriptions() -> dict[str, list[PluginDescription]]: plugins[type_id] = plugins_of_type return plugins -def create_plugin_markdown(plugin: PluginDescription, plugin_type: str, base_dir: Path) -> None: +class RelatedPluginReferenceError(Exception): + """A relatedPlugins reference points at a plugin with no resolvable page.""" + + +class DuplicatePluginIdError(Exception): + """A plugin ID is not unique across the walked plugin types.""" + + +def resolve_related_plugin_links(plugin: PluginDescription, current_plugin_path: str, plugin_paths: dict[str, str]) -> list[tuple[PluginReference, str]]: + """Resolve each related plugin reference to a path relative to the current plugin's own page. + + Raises if any reference points at a plugin with no resolvable page. + """ + current_dir = posixpath.dirname(current_plugin_path) + resolved = [] + unresolvable = [] + for ref in plugin.relatedPlugins: + if ref.id not in plugin_paths: + unresolvable.append(ref.id) + continue + resolved.append((ref, posixpath.relpath(plugin_paths[ref.id], current_dir))) + if unresolvable: + raise RelatedPluginReferenceError( + f"Related plugin(s) {', '.join(unresolvable)} referenced by '{plugin.pluginId}' have no resolvable page" + ) + return resolved + +def create_plugin_markdown(plugin: PluginDescription, base_dir: Path, plugin_paths: dict[str, str]) -> None: """Create markdown document from plugin description.""" if plugin.is_deprecated: click.echo(f"Ignore deprecated plugin {plugin.pluginId}") return click.echo(f"Create reference documentation for {plugin.pluginId}") + current_plugin_path = plugin_paths[plugin.pluginId] + related_plugins_resolved = resolve_related_plugin_links(plugin, current_plugin_path, plugin_paths) + # create content plugin_template = jinja_environment.get_template(f"plugin.md") parameter_template = jinja_environment.get_template(f"parameter.md") @@ -183,15 +214,11 @@ def create_plugin_markdown(plugin: PluginDescription, plugin_type: str, base_dir plugin=plugin, parameters=parameter_content.rstrip("\n"), parameters_advanced=parameter_advanced_content.rstrip("\n"), + related_plugins_resolved=related_plugins_resolved, ) - # create the file (incl. directory) - if plugin.pluginType == "transformer": - directory = base_dir / plugin_type / plugin.main_category - else: - directory = base_dir / plugin_type - directory.mkdir(parents=True, exist_ok=True) - file = directory / f"{plugin.pluginId}.md" + file = base_dir / current_plugin_path + file.parent.mkdir(parents=True, exist_ok=True) with file.open("w", encoding="utf-8") as f: f.write(content) @@ -216,34 +243,33 @@ def create_umbrella_pages(plugins: dict[str, list[PluginDescription]], base_dir: - "Transformers": transformer""" f.write(content) - for plugin_type in plugins: - plugins_of_type = plugins[plugin_type] - if plugin_type == "transformer": + for type_id, plugins_of_type in plugins.items(): + if type_id == "transformer": table_template = jinja_environment.get_template(f"operator_table_with_category.md") else: table_template = jinja_environment.get_template(f"operator_table.md") # Create type-specific index.md file - index_file = base_dir / f"{plugin_type}/index.md" - index_template = jinja_environment.get_template(f"{plugin_type}_base.md") + index_file = base_dir / f"{type_id}/index.md" + index_template = jinja_environment.get_template(f"{type_id}_base.md") items = table_template.render(plugins=plugins_of_type) with index_file.open("w", encoding="utf-8") as f: - click.echo(f"Create {plugin_type} index file in {index_file}") + click.echo(f"Create {type_id} index file in {index_file}") f.write(index_template.render(items=items)) # Create the .pages files - pages_file = base_dir / plugin_type / ".pages" + pages_file = base_dir / type_id / ".pages" pages_content = "nav:\n - index.md" - if plugin_type == "transformer": + if type_id == "transformer": # transformer get separated per main_category - categories = list(set([plugin.main_category for plugin in plugins[plugin_type]])) + categories = list(set([plugin.main_category for plugin in plugins[type_id]])) categories.sort() for category in categories: pages_content += f'\n - "{category}": {category}' - sub_pages_file = base_dir / plugin_type / category / ".pages" + sub_pages_file = base_dir / type_id / category / ".pages" sub_pages_content = "nav:" - category_plugins = [plugin for plugin in plugins[plugin_type] if plugin.main_category == category] + category_plugins = [plugin for plugin in plugins[type_id] if plugin.main_category == category] for plugin in category_plugins: sub_pages_content += f"\n - \"{plugin.title}\": {plugin.pluginId}.md" with sub_pages_file.open("w", encoding="utf-8") as f: @@ -258,6 +284,39 @@ def create_umbrella_pages(plugins: dict[str, list[PluginDescription]], base_dir: click.echo(f"Create .pages file {pages_file}") f.write(pages_content) +def build_plugin_paths(plugins: dict[str, list[PluginDescription]]) -> dict[str, str]: + """Map every non-deprecated plugin's ID to the path its own page will have. + + Deprecated plugins are excluded, since no page is ever generated for them. + """ + plugin_paths: dict[str, str] = {} + for plugins_list in plugins.values(): + for plugin in plugins_list: + if plugin.is_deprecated: + continue + if plugin.pluginType == "transformer": + plugin_paths[plugin.pluginId] = f"{plugin.pluginType}/{plugin.main_category}/{plugin.pluginId}.md" + else: + plugin_paths[plugin.pluginId] = f"{plugin.pluginType}/{plugin.pluginId}.md" + return plugin_paths + +def validate_related_plugin_references(plugins: dict[str, list[PluginDescription]], plugin_paths: dict[str, str]) -> None: + """Raise on every relatedPlugins reference with no resolvable page. + + Deprecated plugins are skipped, since no page is ever generated for + them and their relatedPlugins is otherwise never inspected. + """ + errors = [] + for plugins_list in plugins.values(): + for plugin in plugins_list: + if plugin.is_deprecated: + continue + try: + resolve_related_plugin_links(plugin, plugin_paths[plugin.pluginId], plugin_paths) + except RelatedPluginReferenceError as error: + errors.append(str(error)) + if errors: + raise RelatedPluginReferenceError("\n".join(errors)) @click.command() @click.option( @@ -274,19 +333,23 @@ def update_di_reference(output_dir): click.echo(f"Dump plugins descriptions to {(plugins_json := Path('data/plugins.json'))}") plugins_dump: dict[str, dict] = {} - for category, plugins_list in plugins.items(): + for type_id, plugins_list in plugins.items(): for plugin in plugins_list: if plugin.pluginId in plugins_dump: - raise Exception(f"Duplicate plugin ID: {plugin.pluginId}") + raise DuplicatePluginIdError(f"Duplicate plugin ID: {plugin.pluginId}") plugins_dump[plugin.pluginId] = plugin.model_dump() + + plugin_paths = build_plugin_paths(plugins) + validate_related_plugin_references(plugins, plugin_paths) + plugins_json.write_text(json.dumps(plugins_dump, indent=2)) click.echo(f"Creating DI reference documentation in {basedir}") # create directory structure rmtree(basedir, ignore_errors=True) basedir.mkdir(parents=True, exist_ok=True) - for type_id in plugins: + for type_id, plugins_of_type in plugins.items(): Path(basedir / type_id).mkdir(parents=True, exist_ok=True) - for plugin in plugins[type_id]: - create_plugin_markdown(plugin, type_id, basedir) + for plugin in plugins_of_type: + create_plugin_markdown(plugin, basedir, plugin_paths) create_umbrella_pages(plugins=plugins, base_dir=basedir)