diff --git a/pipeline/src/additional_methods/by_name.py.txt b/pipeline/src/additional_methods/by_name.py.txt index ed875fe0..830ee412 100644 --- a/pipeline/src/additional_methods/by_name.py.txt +++ b/pipeline/src/additional_methods/by_name.py.txt @@ -33,13 +33,17 @@ ignore_separators (bool, optional): Whether to ignore hyphens ("-"), underscores ("_"), slashes ("/"), and repeated whitespace when matching, by collapsing them all to a single space. Defaults to False. + + Raises: + ValueError: if `match` is not one of "equals", "contains" or "within". """ - namelike_properties = ("name", "lookup_label", "family_name", "full_name", "short_name", "abbreviation") + if match not in MATCH_TYPES: + raise ValueError("'match' must be either 'equals', 'contains', or 'within'") if cls._instance_lookup is None: cls._instance_lookup = {} for instance in cls.instances(): keys = [] - for prop_name in namelike_properties: + for prop_name in NAMELIKE_PROPERTIES: value = getattr(instance, prop_name, None) if value is not None: keys.append(value) @@ -52,61 +56,17 @@ else: cls._instance_lookup[key] = [instance] - def remove_accents(s): - import unicodedata - - special = str.maketrans({ - "Ł": "L", "ł": "l", - "Ø": "O", "ø": "o", - "Đ": "D", "đ": "d", - "Ð": "D", "ð": "d", - "Þ": "Th", "þ": "th", - "Æ": "AE", "æ": "ae", - "Œ": "OE", "œ": "oe", - "ß": "ss", "ẞ": "SS", - "Ə": "E", "ə": "e", - "ı": "i", - }) - nfd_form = unicodedata.normalize("NFD", s) - stripped = "".join(c for c in nfd_form if not unicodedata.combining(c)) - return stripped.translate(special) - - def normalize(s): - if not case_sensitive: - s = s.casefold() - if ignore_accents: - s = remove_accents(s) - if ignore_separators: - s = s.replace("-", " ").replace("_", " ").replace("/", " ") - s = " ".join(s.split()) - return s - - if match == "equals": - if case_sensitive and not ignore_accents and not ignore_separators: - matches = cls._instance_lookup.get(name, []) - else: - normalized_name = normalize(name) - matches = [] - for key, instances in cls._instance_lookup.items(): - if normalize(key) == normalized_name: - matches.extend(instances) - elif match == "contains": - normalized_name = normalize(name) - matches = [] - for key, instances in cls._instance_lookup.items(): - if normalized_name in normalize(key): - matches.extend(instances) - elif match == "within": - normalized_name = normalize(name) + if match == "equals" and case_sensitive and not ignore_accents and not ignore_separators: + matches = cls._instance_lookup.get(name, []) # the lookup is keyed on exactly this + else: matches = [] for key, instances in cls._instance_lookup.items(): - if normalize(key) in normalized_name: + if matches_name(key, name, match, case_sensitive, ignore_accents, ignore_separators): matches.extend(instances) - else: - raise ValueError("'match' must be either 'equals', 'contains', or 'within'") if not matches: return None elif all: - return list(dict.fromkeys(matches)) + # de-duplicate by identity, preserving order + return list({id(match): match for match in matches}.values()) else: return matches[0] \ No newline at end of file diff --git a/pipeline/src/base.py b/pipeline/src/base.py index a3e9ed53..70d9fccc 100644 --- a/pipeline/src/base.py +++ b/pipeline/src/base.py @@ -12,6 +12,7 @@ from enum import Enum import json from typing import Union +import unicodedata import rfc3987 @@ -246,6 +247,107 @@ def _resolve_links(self, node_lookup): setattr(self, property.name, resolved_values) +# The properties `by_name()` searches, other than "synonyms", which is list-valued and so is handled separately. +NAMELIKE_PROPERTIES = ("name", "lookup_label", "family_name", "full_name", "short_name", "abbreviation") + +# Letters that `remove_accents()` cannot handle by stripping combining marks, because +# the accent is part of the letter shape or the plain form is more than one letter. +SPECIAL_LETTERS = str.maketrans( + { + "Ł": "L", + "ł": "l", + "Ø": "O", + "ø": "o", + "Đ": "D", + "đ": "d", + "Ð": "D", + "ð": "d", + "Þ": "Th", + "þ": "th", + "Æ": "AE", + "æ": "ae", + "Œ": "OE", + "œ": "oe", + "ß": "ss", + "ẞ": "SS", + "Ə": "E", + "ə": "e", + "ı": "i", + } +) + +MATCH_TYPES = ("equals", "contains", "within") + + +def remove_accents(s): + """ + Strip accents (acute, grave, circumflex) and other diacritical marks (cedilla, tilde, + ring, etc.) from a string, and replace special letters (ß, œ, æ, ø, ł, etc.) by their closest + plain-letter equivalents (e.g. "ß" with "ss"). + """ + nfd_form = unicodedata.normalize("NFD", s) + stripped = "".join(c for c in nfd_form if not unicodedata.combining(c)) + return stripped.translate(SPECIAL_LETTERS) + + +def normalize_name(s, case_sensitive=True, ignore_accents=False, ignore_separators=False): + """ + Put a name-like string into the form in which `matches_name()` compares names. + + Args: + s (str): the string to normalize. + case_sensitive (bool, optional): if False, case-fold the string. Defaults to True. + ignore_accents (bool, optional): if True, apply `remove_accents()`. Defaults to False. + ignore_separators (bool, optional): if True, replace hyphens, underscores and slashes + with spaces, and collapse runs of whitespace to a single space. Defaults to False. + """ + if not case_sensitive: + s = s.casefold() + if ignore_accents: + s = remove_accents(s) + if ignore_separators: + s = s.replace("-", " ").replace("_", " ").replace("/", " ") + s = " ".join(s.split()) + return s + + +def matches_name(value, query, match="equals", case_sensitive=True, ignore_accents=False, + ignore_separators=False): + """ + Whether a name-like property value matches a search string. + + This is the comparison `by_name()` applies to each name in the instance library. + + Args: + value (str): a name-like property value belonging to a metadata node. + query (str): the string being searched for. + match (str, optional): either "equals" (exact match - default), "contains" + (`value` contains `query`), or "within" (`query` contains `value`). + case_sensitive (bool, optional): Whether the comparison should be case-sensitive. + Defaults to True. + ignore_accents (bool, optional): Whether to ignore accents (acute, grave, circumflex) + and other diacritical marks (cedilla, tilde, ring, etc.) when matching. Also treat + special letters (ß, œ, æ, ø, ł, etc.) as their closest plain-letter equivalents + (e.g. "ß" as "ss"). Defaults to False. + ignore_separators (bool, optional): Whether to ignore hyphens ("-"), underscores ("_"), + slashes ("/"), and repeated whitespace when matching, by collapsing them all to a + single space. Defaults to False. + + Raises: + ValueError: if `match` is not one of "equals", "contains" or "within". + """ + normalized_value = normalize_name(value, case_sensitive, ignore_accents, ignore_separators) + normalized_query = normalize_name(query, case_sensitive, ignore_accents, ignore_separators) + if match == "equals": + return normalized_value == normalized_query + elif match == "contains": + return normalized_query in normalized_value + elif match == "within": + return normalized_value in normalized_query + else: + raise ValueError("'match' must be either 'equals', 'contains', or 'within'") + + class LinkedMetadata(Node): """ A Python representation of a metadata node that should have a unique identifier. diff --git a/pipeline/src/module_template.py.txt b/pipeline/src/module_template.py.txt index f565a157..f036854c 100644 --- a/pipeline/src/module_template.py.txt +++ b/pipeline/src/module_template.py.txt @@ -6,7 +6,7 @@ {{preamble}} -from openminds.base import {{ base_class }} +from openminds.base import {{ base_class }}{% if additional_methods %}, MATCH_TYPES, NAMELIKE_PROPERTIES, matches_name{% endif %} from openminds.properties import Property diff --git a/pipeline/tests/test_name_matching.py b/pipeline/tests/test_name_matching.py new file mode 100644 index 00000000..729af613 --- /dev/null +++ b/pipeline/tests/test_name_matching.py @@ -0,0 +1,109 @@ +""" +Tests for the name-matching helpers in `openminds.base`, +and for the parts of the `by_name()` contract that depend on them. +""" + +import pytest + +import openminds.latest +import openminds.v4 +from openminds.base import ( + MATCH_TYPES, + NAMELIKE_PROPERTIES, + matches_name, + normalize_name, + remove_accents, +) + + +@pytest.mark.parametrize( + "text,expected", + [ + ("Müller", "Muller"), + ("République française", "Republique francaise"), + ("Azərbaycan Respublikası", "Azerbaycan Respublikasi"), + ("Straße", "Strasse"), + ("Œuvre", "OEuvre"), + ("Ångström", "Angstrom"), + ("plain text", "plain text"), + ], +) +def test_remove_accents(text, expected): + assert remove_accents(text) == expected + + +def test_normalize_name(): + assert normalize_name("Raphé") == "Raphé" + assert normalize_name("Raphé", case_sensitive=False) == "raphé" + assert normalize_name("Raphé", ignore_accents=True) == "Raphe" + assert normalize_name("Raphé", case_sensitive=False, ignore_accents=True) == "raphe" + assert normalize_name("CLARITY/TDE") == "CLARITY/TDE" + assert normalize_name("CLARITY/TDE", ignore_separators=True) == "CLARITY TDE" + assert normalize_name("two-photon imaging", ignore_separators=True) == "two photon imaging" + + +class TestMatchesName: + def test_equals_is_case_sensitive_by_default(self): + assert matches_name("Mus musculus", "Mus musculus") + assert not matches_name("Mus musculus", "mus musculus") + assert matches_name("Mus musculus", "mus musculus", case_sensitive=False) + + def test_accents(self): + assert not matches_name("République française", "Republique francaise") + assert matches_name("République française", "Republique francaise", ignore_accents=True) + assert matches_name("Republique francaise", "République française", ignore_accents=True) + + def test_contains(self): + assert matches_name("Mus musculus", "musculus", match="contains") + assert not matches_name("musculus", "Mus musculus", match="contains") + + def test_within(self): + assert matches_name("Mus musculus", "Mus musculus - House mouse", match="within") + assert not matches_name("Mus musculus - House mouse", "Mus musculus", match="within") + + def test_separators(self): + assert not matches_name("CLARITY/TDE", "CLARITY-TDE") + assert matches_name("CLARITY/TDE", "CLARITY-TDE", ignore_separators=True) + assert matches_name("CLARITY/TDE", "CLARITY TDE", ignore_separators=True) + assert matches_name("two-photon fluorescence microscopy", "two photon fluorescence microscopy", + ignore_separators=True) + + def test_invalid_match(self): + with pytest.raises(ValueError): + matches_name("a", "a", match="approximately") + + def test_match_types(self): + assert MATCH_TYPES == ("equals", "contains", "within") + + +@pytest.mark.parametrize("om", [openminds.latest]) +class TestByNameUsesMatchesName: + """`by_name()` must apply exactly the rules `matches_name()` describes.""" + + @pytest.mark.parametrize("match", MATCH_TYPES) + @pytest.mark.parametrize("case_sensitive", [True, False]) + @pytest.mark.parametrize("ignore_accents", [True, False]) + @pytest.mark.parametrize("ignore_separators", [True, False]) + def test_agreement(self, om, match, case_sensitive, ignore_accents, ignore_separators): + SovereignState = om.controlled_terms.SovereignState + query = "Republique francaise" + found = SovereignState.by_name( + query, + match=match, + all=True, + case_sensitive=case_sensitive, + ignore_accents=ignore_accents, + ignore_separators=ignore_separators, + ) + for state in found or []: + names = [getattr(state, prop_name, None) for prop_name in NAMELIKE_PROPERTIES] + names += list(state.synonyms or []) if hasattr(state, "synonyms") else [] + assert any( + matches_name(name, query, match, case_sensitive, ignore_accents, ignore_separators) + for name in names + if name is not None + ) + + def test_invalid_match_is_rejected(self, om): + with pytest.raises(ValueError): + om.controlled_terms.Species.by_name("Mus musculus", match="approximately")