Hoist by_name() matching helpers out of template into importable code

pipeline/src/additional_methods/by_name.py.txt

@@ -33,13 +33,17 @@
             ignore_separators (bool, optional): Whether to ignore hyphens ("-"), underscores
                 ("_"), slashes ("/"), and repeated whitespace when matching, by collapsing
                 them all to a single space. Defaults to False.
+
+        Raises:
+            ValueError: if `match` is not one of "equals", "contains" or "within".
         """
-        namelike_properties = ("name", "lookup_label", "family_name", "full_name", "short_name", "abbreviation")
+        if match not in MATCH_TYPES:
+            raise ValueError("'match' must be either 'equals', 'contains', or 'within'")
         if cls._instance_lookup is None:
             cls._instance_lookup = {}
             for instance in cls.instances():
                 keys = []
-                for prop_name in namelike_properties:
+                for prop_name in NAMELIKE_PROPERTIES:
                     value = getattr(instance, prop_name, None)
                     if value is not None:
                         keys.append(value)
@@ -52,61 +56,17 @@
                     else:
                         cls._instance_lookup[key] = [instance]
 
-        def remove_accents(s):
-            import unicodedata
-
-            special = str.maketrans({
-                "Ł": "L", "ł": "l",
-                "Ø": "O", "ø": "o",
-                "Đ": "D", "đ": "d",
-                "Ð": "D", "ð": "d",
-                "Þ": "Th", "þ": "th",
-                "Æ": "AE", "æ": "ae",
-                "Œ": "OE", "œ": "oe",
-                "ß": "ss", "ẞ": "SS",
-                "Ə": "E", "ə": "e",
-                "ı": "i",
-            })
-            nfd_form = unicodedata.normalize("NFD", s)
-            stripped = "".join(c for c in nfd_form if not unicodedata.combining(c))
-            return stripped.translate(special)
-
-        def normalize(s):
-            if not case_sensitive:
-                s = s.casefold()
-            if ignore_accents:
-                s = remove_accents(s)
-            if ignore_separators:
-                s = s.replace("-", " ").replace("_", " ").replace("/", " ")
-                s = " ".join(s.split())
-            return s
-
-        if match == "equals":
-            if case_sensitive and not ignore_accents and not ignore_separators:
-                matches = cls._instance_lookup.get(name, [])
-            else:
-                normalized_name = normalize(name)
-                matches = []
-                for key, instances in cls._instance_lookup.items():
-                    if normalize(key) == normalized_name:
-                        matches.extend(instances)
-        elif match == "contains":
-            normalized_name = normalize(name)
-            matches = []
-            for key, instances in cls._instance_lookup.items():
-                if normalized_name in normalize(key):
-                    matches.extend(instances)
-        elif match == "within":
-            normalized_name = normalize(name)
+        if match == "equals" and case_sensitive and not ignore_accents and not ignore_separators:
+            matches = cls._instance_lookup.get(name, [])  # the lookup is keyed on exactly this
+        else:
             matches = []
             for key, instances in cls._instance_lookup.items():
-                if normalize(key) in normalized_name:
+                if matches_name(key, name, match, case_sensitive, ignore_accents, ignore_separators):
                     matches.extend(instances)
-        else:
-            raise ValueError("'match' must be either 'equals', 'contains', or 'within'")
         if not matches:
             return None
         elif all:
-            return list(dict.fromkeys(matches))
+            # de-duplicate by identity, preserving order
+            return list({id(match): match for match in matches}.values())
         else:
             return matches[0]

pipeline/src/base.py

@@ -12,6 +12,7 @@
 from enum import Enum
 import json
 from typing import Union
+import unicodedata
 
 import rfc3987
 
@@ -246,6 +247,107 @@ def _resolve_links(self, node_lookup):
                 setattr(self, property.name, resolved_values)
 
 
+# The properties `by_name()` searches, other than "synonyms", which is list-valued and so is handled separately.
+NAMELIKE_PROPERTIES = ("name", "lookup_label", "family_name", "full_name", "short_name", "abbreviation")
+
+# Letters that `remove_accents()` cannot handle by stripping combining marks, because
+# the accent is part of the letter shape or the plain form is more than one letter.
+SPECIAL_LETTERS = str.maketrans(
+    {
+        "Ł": "L",
+        "ł": "l",
+        "Ø": "O",
+        "ø": "o",
+        "Đ": "D",
+        "đ": "d",
+        "Ð": "D",
+        "ð": "d",
+        "Þ": "Th",
+        "þ": "th",
+        "Æ": "AE",
+        "æ": "ae",
+        "Œ": "OE",
+        "œ": "oe",
+        "ß": "ss",
+        "ẞ": "SS",
+        "Ə": "E",
+        "ə": "e",
+        "ı": "i",
+    }
+)
+
+MATCH_TYPES = ("equals", "contains", "within")
+
+
+def remove_accents(s):
+    """
+    Strip accents (acute, grave, circumflex) and other diacritical marks (cedilla, tilde,
+    ring, etc.) from a string, and replace special letters (ß, œ, æ, ø, ł, etc.) by their closest
+    plain-letter equivalents (e.g. "ß" with "ss").
+    """
+    nfd_form = unicodedata.normalize("NFD", s)
+    stripped = "".join(c for c in nfd_form if not unicodedata.combining(c))
+    return stripped.translate(SPECIAL_LETTERS)
+
+
+def normalize_name(s, case_sensitive=True, ignore_accents=False, ignore_separators=False):
+    """
+    Put a name-like string into the form in which `matches_name()` compares names.
+
+    Args:
+        s (str): the string to normalize.
+        case_sensitive (bool, optional): if False, case-fold the string. Defaults to True.
+        ignore_accents (bool, optional): if True, apply `remove_accents()`. Defaults to False.
+        ignore_separators (bool, optional): if True, replace hyphens, underscores and slashes
+            with spaces, and collapse runs of whitespace to a single space. Defaults to False.
+    """
+    if not case_sensitive:
+        s = s.casefold()
+    if ignore_accents:
+        s = remove_accents(s)
+    if ignore_separators:
+        s = s.replace("-", " ").replace("_", " ").replace("/", " ")
+        s = " ".join(s.split())
+    return s
+
+
+def matches_name(value, query, match="equals", case_sensitive=True, ignore_accents=False,
+                 ignore_separators=False):
+    """
+    Whether a name-like property value matches a search string.
+
+    This is the comparison `by_name()` applies to each name in the instance library.
+
+    Args:
+        value (str): a name-like property value belonging to a metadata node.
+        query (str): the string being searched for.
+        match (str, optional): either "equals" (exact match - default), "contains"
+            (`value` contains `query`), or "within" (`query` contains `value`).
+        case_sensitive (bool, optional): Whether the comparison should be case-sensitive.
+            Defaults to True.
+        ignore_accents (bool, optional): Whether to ignore accents (acute, grave, circumflex)
+            and other diacritical marks (cedilla, tilde, ring, etc.) when matching. Also treat
+            special letters (ß, œ, æ, ø, ł, etc.) as their closest plain-letter equivalents
+            (e.g. "ß" as "ss"). Defaults to False.
+        ignore_separators (bool, optional): Whether to ignore hyphens ("-"), underscores ("_"),
+            slashes ("/"), and repeated whitespace when matching, by collapsing them all to a
+            single space. Defaults to False.
+
+    Raises:
+        ValueError: if `match` is not one of "equals", "contains" or "within".
+    """
+    normalized_value = normalize_name(value, case_sensitive, ignore_accents, ignore_separators)
+    normalized_query = normalize_name(query, case_sensitive, ignore_accents, ignore_separators)
+    if match == "equals":
+        return normalized_value == normalized_query
+    elif match == "contains":
+        return normalized_query in normalized_value
+    elif match == "within":
+        return normalized_value in normalized_query
+    else:
+        raise ValueError("'match' must be either 'equals', 'contains', or 'within'")
+
+
 class LinkedMetadata(Node):
     """
     A Python representation of a metadata node that should have a unique identifier.

pipeline/src/module_template.py.txt

@@ -6,7 +6,7 @@
 
 {{preamble}}
 
-from openminds.base import {{ base_class }}
+from openminds.base import {{ base_class }}{% if additional_methods %}, MATCH_TYPES, NAMELIKE_PROPERTIES, matches_name{% endif %}
 from openminds.properties import Property
 
 

pipeline/tests/test_name_matching.py

@@ -0,0 +1,109 @@
+"""
+Tests for the name-matching helpers in `openminds.base`, 
+and for the parts of the `by_name()` contract that depend on them.
+"""
+
+import pytest
+
+import openminds.latest
+import openminds.v4
+from openminds.base import (
+    MATCH_TYPES,
+    NAMELIKE_PROPERTIES,
+    matches_name,
+    normalize_name,
+    remove_accents,
+)
+
+
+@pytest.mark.parametrize(
+    "text,expected",
+    [
+        ("Müller", "Muller"),
+        ("République française", "Republique francaise"),
+        ("Azərbaycan Respublikası", "Azerbaycan Respublikasi"),
+        ("Straße", "Strasse"),
+        ("Œuvre", "OEuvre"),
+        ("Ångström", "Angstrom"),
+        ("plain text", "plain text"),
+    ],
+)
+def test_remove_accents(text, expected):
+    assert remove_accents(text) == expected
+
+
+def test_normalize_name():
+    assert normalize_name("Raphé") == "Raphé"
+    assert normalize_name("Raphé", case_sensitive=False) == "raphé"
+    assert normalize_name("Raphé", ignore_accents=True) == "Raphe"
+    assert normalize_name("Raphé", case_sensitive=False, ignore_accents=True) == "raphe"
+    assert normalize_name("CLARITY/TDE") == "CLARITY/TDE"
+    assert normalize_name("CLARITY/TDE", ignore_separators=True) == "CLARITY TDE"
+    assert normalize_name("two-photon  imaging", ignore_separators=True) == "two photon imaging"
+
+
+class TestMatchesName:
+    def test_equals_is_case_sensitive_by_default(self):
+        assert matches_name("Mus musculus", "Mus musculus")
+        assert not matches_name("Mus musculus", "mus musculus")
+        assert matches_name("Mus musculus", "mus musculus", case_sensitive=False)
+
+    def test_accents(self):
+        assert not matches_name("République française", "Republique francaise")
+        assert matches_name("République française", "Republique francaise", ignore_accents=True)
+        assert matches_name("Republique francaise", "République française", ignore_accents=True)
+
+    def test_contains(self):
+        assert matches_name("Mus musculus", "musculus", match="contains")
+        assert not matches_name("musculus", "Mus musculus", match="contains")
+
+    def test_within(self):
+        assert matches_name("Mus musculus", "Mus musculus - House mouse", match="within")
+        assert not matches_name("Mus musculus - House mouse", "Mus musculus", match="within")
+
+    def test_separators(self):
+        assert not matches_name("CLARITY/TDE", "CLARITY-TDE")
+        assert matches_name("CLARITY/TDE", "CLARITY-TDE", ignore_separators=True)
+        assert matches_name("CLARITY/TDE", "CLARITY   TDE", ignore_separators=True)
+        assert matches_name("two-photon fluorescence microscopy", "two photon fluorescence microscopy",
+                            ignore_separators=True)
+
+    def test_invalid_match(self):
+        with pytest.raises(ValueError):
+            matches_name("a", "a", match="approximately")
+
+    def test_match_types(self):
+        assert MATCH_TYPES == ("equals", "contains", "within")
+
+
+@pytest.mark.parametrize("om", [openminds.latest])
+class TestByNameUsesMatchesName:
+    """`by_name()` must apply exactly the rules `matches_name()` describes."""
+
+    @pytest.mark.parametrize("match", MATCH_TYPES)
+    @pytest.mark.parametrize("case_sensitive", [True, False])
+    @pytest.mark.parametrize("ignore_accents", [True, False])
+    @pytest.mark.parametrize("ignore_separators", [True, False])
+    def test_agreement(self, om, match, case_sensitive, ignore_accents, ignore_separators):
+        SovereignState = om.controlled_terms.SovereignState
+        query = "Republique francaise"
+        found = SovereignState.by_name(
+            query,
+            match=match,
+            all=True,
+            case_sensitive=case_sensitive,
+            ignore_accents=ignore_accents,
+            ignore_separators=ignore_separators,
+        )
+        for state in found or []:
+            names = [getattr(state, prop_name, None) for prop_name in NAMELIKE_PROPERTIES]
+            names += list(state.synonyms or []) if hasattr(state, "synonyms") else []
+            assert any(
+                matches_name(name, query, match, case_sensitive, ignore_accents, ignore_separators)
+                for name in names
+                if name is not None
+            )
+
+    def test_invalid_match_is_rejected(self, om):
+        with pytest.raises(ValueError):
+            om.controlled_terms.Species.by_name("Mus musculus", match="approximately")