From 0b6b47b80e14d98d2a69e6392b351fc6d9dece8b Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 18:33:20 +0200 Subject: [PATCH 01/10] BOF: local JSONID registry From 4f81758d6bc062e2f057c2226a3e0a085628a48e Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 19:30:34 +0200 Subject: [PATCH 02/10] Update .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 6071354..8e77aea 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,7 @@ jsonid-integration-files/ # Secreta token.pypi jsonid_pronom.xml +*.csv +*.json +*.diff +token-pypi From 6ec6e4c2f2a9f33980097f9e2d87f82f4c666ad3 Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 19:30:57 +0200 Subject: [PATCH 03/10] WIP: local registry --- src/jsonid/jsonid.py | 13 +++++++++++++ src/jsonid/local.py | 7 +++++++ 2 files changed, 20 insertions(+) create mode 100644 src/jsonid/local.py diff --git a/src/jsonid/jsonid.py b/src/jsonid/jsonid.py index 9d3379e..8f64dad 100644 --- a/src/jsonid/jsonid.py +++ b/src/jsonid/jsonid.py @@ -216,9 +216,18 @@ def main() -> None: ) parser.add_argument( "--registry", + "--local", help="path to a custom registry to lead into memory replacing the default", required=False, ) + parser.add_argument( + "--localonly", + "--lonly", + "--lonely", + help="if a local registry is specified, use this and this only", + required=False, + action="store_true", + ) # NB. consider output to stdout once the feature is more stable. parser.add_argument( "--pronom", @@ -285,6 +294,8 @@ def main() -> None: # Primary application functions. if args.registry: raise NotImplementedError("custom registry is not yet available") + if args.local_only: + raise NotImplementedError("todo...") if args.pronom: export.export_pronom() sys.exit() @@ -305,6 +316,8 @@ def main() -> None: logger.info("ok") sys.exit() if args.html: + if args.registry: + raise NotImplementedError("local registry output is not yet supported") helpers.html() sys.exit() if not strategy: diff --git a/src/jsonid/local.py b/src/jsonid/local.py new file mode 100644 index 0000000..ba78e0f --- /dev/null +++ b/src/jsonid/local.py @@ -0,0 +1,7 @@ +"""Functions supporting local registry use""" + + +def load_local_registry(): + """Load the local registry and return it as a data structure + to the caller. + """ From 05122dbfee505b10b71e9e843834683b82615fab Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 19:31:07 +0200 Subject: [PATCH 04/10] WIP: add local registry structure --- local/registry.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 local/registry.toml diff --git a/local/registry.toml b/local/registry.toml new file mode 100644 index 0000000..16fd85b --- /dev/null +++ b/local/registry.toml @@ -0,0 +1,19 @@ +[[entries]] + +name = "doctype1" +identifier = "local0001" + +[[doctype.markers]] + +key = "key1" +is = "value1" + +[[entries]] + +name = "doctype2" +identifier = "local0002" + +[[doctype.markers]] + +key = "key2" +is = "value2" From 5ef053ada8649beff3e9e919c7530abe88233b8a Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 22:40:39 +0200 Subject: [PATCH 05/10] WIP: add test harness --- src/jsonid/jsonid.py | 11 ++++++---- src/jsonid/local.py | 33 +++++++++++++++++++++++++++++- tests/test_local_registry.py | 39 ++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 tests/test_local_registry.py diff --git a/src/jsonid/jsonid.py b/src/jsonid/jsonid.py index 8f64dad..9f62c2c 100644 --- a/src/jsonid/jsonid.py +++ b/src/jsonid/jsonid.py @@ -16,11 +16,12 @@ import helpers import lookup import registry + import local except ModuleNotFoundError: try: - from src.jsonid import export, file_processing, helpers, lookup, registry + from src.jsonid import export, file_processing, helpers, lookup, registry, local except ModuleNotFoundError: - from jsonid import export, file_processing, helpers, lookup, registry + from jsonid import export, file_processing, helpers, lookup, registry, local logger = None @@ -293,9 +294,11 @@ def main() -> None: # Primary application functions. if args.registry: - raise NotImplementedError("custom registry is not yet available") - if args.local_only: + local.load_and_parse_local_registry(path=args.registry) + if args.localonly: raise NotImplementedError("todo...") + + return if args.pronom: export.export_pronom() sys.exit() diff --git a/src/jsonid/local.py b/src/jsonid/local.py index ba78e0f..2c99ca5 100644 --- a/src/jsonid/local.py +++ b/src/jsonid/local.py @@ -1,7 +1,38 @@ """Functions supporting local registry use""" +import logging +import pathlib +import tomllib as toml -def load_local_registry(): +logger = logging.getLogger(__name__) + + +class LocalRegistryException(Exception): + """Exception to raise if something goes wrong with the local + registry. + """ + + +def load_and_parse_local_registry(path: str): + """Read the data use the data.""" + + registry = pathlib.Path(path) + if not registry.exists(): + raise LocalRegistryException("registry path not found") + + load_local_registry(registry) + + +def load_local_registry(registry: pathlib.Path): """Load the local registry and return it as a data structure to the caller. """ + + with registry.open() as data: + registry_data = data.read() + + reg = toml.loads(registry_data) + + logger.debug("local registry length: %d", len(reg["entries"])) + + assert False diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py new file mode 100644 index 0000000..72e2e68 --- /dev/null +++ b/tests/test_local_registry.py @@ -0,0 +1,39 @@ +"""Test functions associated with the local registry.""" + +from src.jsonid import local + +from typing import Final + +registry: Final[str] = """ +[[entries]] + +name = "doctype1" +identifier = "local0001" + +[[entries.markers]] + +key = "key1" +is = "value1" + +[[entries]] + +name = "doctype2" +identifier = "local0002" + +[[entries.markers]] + +key = "key2" +is = "value2" + +""" + +import io + + +def test_load_local(tmp_path): + """Ensure loading the local registry works as anticipated.""" + + a = tmp_path / "registry_path" + a.write_text(registry) + + local.load_local_registry(a) From 5e127976a5f3615c3b7691762c60865274c33c38 Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sun, 9 Aug 2026 11:15:32 +0200 Subject: [PATCH 06/10] WIP: README and other pieces... --- README.md | 34 +++++++++++++++++++++++++++++++--- src/jsonid/jsonid.py | 5 +++-- src/jsonid/local.py | 3 +++ tests/test_local_registry.py | 9 +++++---- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1a23ed0..1e21bfa 100644 --- a/README.md +++ b/README.md @@ -405,9 +405,37 @@ within the `sops` object/value. ### Local rules -The plan is to allow local rules to be run alongside the global ruleset. I -expect this will be a bit further down the line when the ruleset and -metaddata is more stabilised. +You can define local rules in a local registry object. Local registries are +defined in TOML, and look as follows: + +```toml +[[entries]] + +name = "doctype1" +identifier = "local0001" + +[[entries.markers]] + +key = "key1" +is = "value1" + +[[entries]] + +name = "doctype2" +identifier = "local0002" + +[[entries.markers]] + +key = "key2" +is = "value2" +``` + +Markers follow the same pattern as the standard registry. + +#### Local only + +Use the `--localonly` flag to use _just_ your custom markers in your format +identification workflow. ## PRONOM diff --git a/src/jsonid/jsonid.py b/src/jsonid/jsonid.py index 9f62c2c..a148643 100644 --- a/src/jsonid/jsonid.py +++ b/src/jsonid/jsonid.py @@ -16,12 +16,13 @@ import helpers import lookup import registry + import local except ModuleNotFoundError: try: - from src.jsonid import export, file_processing, helpers, lookup, registry, local + from src.jsonid import export, file_processing, helpers, local, lookup, registry except ModuleNotFoundError: - from jsonid import export, file_processing, helpers, lookup, registry, local + from jsonid import export, file_processing, helpers, local, lookup, registry logger = None diff --git a/src/jsonid/local.py b/src/jsonid/local.py index 2c99ca5..17a8c17 100644 --- a/src/jsonid/local.py +++ b/src/jsonid/local.py @@ -35,4 +35,7 @@ def load_local_registry(registry: pathlib.Path): logger.debug("local registry length: %d", len(reg["entries"])) + for item in reg["entries"]: + print(item) + assert False diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py index 72e2e68..280c88d 100644 --- a/tests/test_local_registry.py +++ b/tests/test_local_registry.py @@ -1,10 +1,12 @@ """Test functions associated with the local registry.""" -from src.jsonid import local - from typing import Final -registry: Final[str] = """ +from src.jsonid import local + +registry: Final[ + str +] = """ [[entries]] name = "doctype1" @@ -27,7 +29,6 @@ """ -import io def test_load_local(tmp_path): From 3fa182da65a4cf7ce188b8f547a72e88adcd8225 Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sun, 9 Aug 2026 11:28:19 +0200 Subject: [PATCH 07/10] WIP: look at testing optional entries NB. localref is an ID or URI local to the registry. --- README.md | 2 ++ local/registry.toml | 7 +++++-- tests/test_local_registry.py | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1e21bfa..837cc80 100644 --- a/README.md +++ b/README.md @@ -413,6 +413,7 @@ defined in TOML, and look as follows: name = "doctype1" identifier = "local0001" +localref = "http://example.com/repository/ID" [[entries.markers]] @@ -423,6 +424,7 @@ is = "value1" name = "doctype2" identifier = "local0002" +#localref = "http://example.com/repository/ID [[entries.markers]] diff --git a/local/registry.toml b/local/registry.toml index 16fd85b..aff854d 100644 --- a/local/registry.toml +++ b/local/registry.toml @@ -3,7 +3,10 @@ name = "doctype1" identifier = "local0001" -[[doctype.markers]] +# optional +localref = "http://example.com/repository/ID + +[[entries.markers]] key = "key1" is = "value1" @@ -13,7 +16,7 @@ is = "value1" name = "doctype2" identifier = "local0002" -[[doctype.markers]] +[[entries.markers]] key = "key2" is = "value2" diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py index 280c88d..03a02dd 100644 --- a/tests/test_local_registry.py +++ b/tests/test_local_registry.py @@ -30,7 +30,6 @@ """ - def test_load_local(tmp_path): """Ensure loading the local registry works as anticipated.""" From 77fef60db5cbb52fd3c8831327d0b12ac473e88f Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Wed, 12 Aug 2026 00:23:14 +0200 Subject: [PATCH 08/10] WIP: slow progress but progress... --- README.md | 4 +- local/registry.toml | 9 +++-- src/jsonid/local.py | 75 +++++++++++++++++++++++++++++++++--- tests/test_local_registry.py | 5 +++ 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 837cc80..cacd77d 100644 --- a/README.md +++ b/README.md @@ -413,7 +413,7 @@ defined in TOML, and look as follows: name = "doctype1" identifier = "local0001" -localref = "http://example.com/repository/ID" +localref = "http://example.com/doctype/spec/ID" [[entries.markers]] @@ -424,7 +424,7 @@ is = "value1" name = "doctype2" identifier = "local0002" -#localref = "http://example.com/repository/ID +localref = "http://example.com/doctype/spec/ID [[entries.markers]] diff --git a/local/registry.toml b/local/registry.toml index aff854d..a0c36eb 100644 --- a/local/registry.toml +++ b/local/registry.toml @@ -2,15 +2,18 @@ name = "doctype1" identifier = "local0001" - -# optional -localref = "http://example.com/repository/ID +localref = "http://example.com/repository/ID" [[entries.markers]] key = "key1" is = "value1" +[[entries.markers]] + +key = "key2" +is = "value2" + [[entries]] name = "doctype2" diff --git a/src/jsonid/local.py b/src/jsonid/local.py index 17a8c17..585e25b 100644 --- a/src/jsonid/local.py +++ b/src/jsonid/local.py @@ -4,6 +4,15 @@ import pathlib import tomllib as toml +try: + import registry_class + import registry_data +except ModuleNotFoundError: + try: + from src.jsonid import registry_class, registry_data + except ModuleNotFoundError: + from jsonid import registry_class, registry_data + logger = logging.getLogger(__name__) @@ -23,19 +32,75 @@ def load_and_parse_local_registry(path: str): load_local_registry(registry) +""" + registry_class.RegistryEntry( + identifier="jrid:0001", + name=[{"@en": "JavaScript Package Lock"}], + description=[{"@en": "describes an exact Node (NPM) module dependency tree"}], + markers=[ + {"KEY": "name", "EXISTS": None}, + {"KEY": "lockfileVersion", "EXISTS": None}, + {"KEY": "packages", "EXISTS": None}, + ], + ), +""" + + def load_local_registry(registry: pathlib.Path): """Load the local registry and return it as a data structure to the caller. """ with registry.open() as data: - registry_data = data.read() + local_registry_data = data.read() + + local_reg = toml.loads(local_registry_data) + + logger.debug("local registry length: %d", len(local_reg["entries"])) + + reg = registry_data.registry() + + for item in local_reg["entries"]: + + # TODO: cleanup, ensure keys are capitalized. + m = [] + for i in item["markers"]: + d = {} + for k, v in tuple(i.items()): + d.update({k.upper(): v}) + m.append(d) + + # TODO: variable naming. + a = registry_class.RegistryEntry( + identifier=item["identifier"], + name=[{"@en": "TODO"}], + description=[{"@en": "TODO"}], + markers=m, + ) - reg = toml.loads(registry_data) + print(a) + print(a.markers) + print("---") + # if local... + reg.append(a) - logger.debug("local registry length: %d", len(reg["entries"])) + # print(reg) - for item in reg["entries"]: - print(item) + """ + [[entries]] + + name = "doctype1" + identifier = "local0001" + localref = "http://example.com/repository/ID + + [[entries.markers]] + + key = "key1" + is = "value1" + + {'name': 'doctype1', 'identifier': 'local0001', 'markers': [{'key': 'key1', 'is': 'value1'}]} + + + """ assert False diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py index 03a02dd..82559b2 100644 --- a/tests/test_local_registry.py +++ b/tests/test_local_registry.py @@ -17,6 +17,11 @@ key = "key1" is = "value1" +[[entries.markers]] + +key = "key2" +is = "value2" + [[entries]] name = "doctype2" From 6a64eb0481278ad1b25accbb511a03ea459cf348 Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Mon, 7 Sep 2026 23:48:22 +0200 Subject: [PATCH 09/10] WIP: more progress... * tests getting there... * checks for symlinks added for resillience... * language will need to be looked at in a separate feature--this should be fine as @en is still an opaque field --- src/jsonid/file_processing.py | 52 +++++++++++++++++++++-------- src/jsonid/jsonid.py | 33 +++++++++++++++---- src/jsonid/local.py | 39 ++++++++-------------- src/jsonid/registry.py | 16 ++++++--- tests/test_local_registry.py | 62 +++++++++++++++++++++++++++++------ 5 files changed, 142 insertions(+), 60 deletions(-) diff --git a/src/jsonid/file_processing.py b/src/jsonid/file_processing.py index fa5927e..ff888d4 100644 --- a/src/jsonid/file_processing.py +++ b/src/jsonid/file_processing.py @@ -164,6 +164,9 @@ async def analyse_json(paths: list[str], strategy: list): """Analyse a JSON object.""" analysis_res = [] for path in paths: + if os.path.islink(path): + logger.debug(f"'{path}' is a symlink") + continue if os.path.getsize(path) == 0: logger.debug("%s is an empty file", path) continue @@ -200,8 +203,13 @@ async def process_result( base_obj: registry.BaseCharacteristics, padding: int, agentout: bool, + reg_data: list, ): - """Process something JSON/YAML/TOML""" + """Process something JSON/YAML/TOML. + + TODO: ... + + """ results = [] # NB. these switch-like ifs might not be needed in the fullness # of time. It depends if we need to do any custom processing of @@ -225,21 +233,23 @@ async def process_result( return # If we don't exit early and we try and identify the file... we then # create a new class object with an identification... - if base_obj.doctype == registry.DOCTYPE_JSON: - results = registry.matcher(base_obj) - if base_obj.doctype == registry.DOCTYPE_JSONL: - results = registry.matcher(base_obj) - if base_obj.doctype == registry.DOCTYPE_YAML: - results = registry.matcher(base_obj) - if base_obj.doctype == registry.DOCTYPE_TOML: - results = registry.matcher(base_obj) + if base_obj.doctype not in ( + registry.DOCTYPE_JSON, + registry.DOCTYPE_JSONL, + registry.DOCTYPE_YAML, + registry.DOCTYPE_TOML, + ): + return + results = registry.matcher( + base_obj=base_obj, + reg_data=reg_data, + ) output.output_results( path=path, results=results, padding=padding, agentout=agentout, ) - return def _get_padding(paths: list): @@ -255,10 +265,18 @@ def _get_padding(paths: list): return padding -async def identify_json(paths: list[str], strategy: list, binary: bool, agentout: bool): - """Identify objects.""" +async def identify_json( + paths: list[str], strategy: list, binary: bool, agentout: bool, reg_data: list +): + """Identify objects. + + TODO: ,,, + """ padding = _get_padding(paths=paths) for _, path in enumerate(paths): + if os.path.islink(path): + logger.debug(f"'{path}' is a symlink") + continue if os.path.getsize(path) == 0: logger.debug("%s is an empty file", path) base_obj = registry.BaseCharacteristics(empty=True) @@ -268,6 +286,7 @@ async def identify_json(paths: list[str], strategy: list, binary: bool, agentout base_obj=base_obj, padding=padding, agentout=agentout, + reg_data=reg_data, ) continue base_obj = await identify_plaintext_bytestream( @@ -283,6 +302,7 @@ async def identify_json(paths: list[str], strategy: list, binary: bool, agentout base_obj=base_obj, padding=padding, agentout=agentout, + reg_data=reg_data, ) continue logger.debug("processing: %s (%s)", path, base_obj.doctype) @@ -291,6 +311,7 @@ async def identify_json(paths: list[str], strategy: list, binary: bool, agentout base_obj=base_obj, padding=padding, agentout=agentout, + reg_data=reg_data, ) @@ -443,7 +464,9 @@ async def process_glob(glob_path: str): return paths -async def process_data(path: str, strategy: list, binary: bool, agentout: bool): +async def process_data( + path: str, strategy: list, binary: bool, agentout: bool, reg_data: list +): """Process all objects at a given path.""" logger.debug("processing: %s", path) if "*" in path: @@ -453,6 +476,7 @@ async def process_data(path: str, strategy: list, binary: bool, agentout: bool): strategy=strategy, binary=binary, agentout=agentout, + reg_data=reg_data, ) sys.exit(0) if not os.path.exists(path): @@ -464,6 +488,7 @@ async def process_data(path: str, strategy: list, binary: bool, agentout: bool): strategy=strategy, binary=binary, agentout=agentout, + reg_data=reg_data, ) sys.exit(0) paths = await create_manifest(path) @@ -475,6 +500,7 @@ async def process_data(path: str, strategy: list, binary: bool, agentout: bool): strategy=strategy, binary=binary, agentout=agentout, + reg_data=reg_data, ) diff --git a/src/jsonid/jsonid.py b/src/jsonid/jsonid.py index a148643..eb64a2d 100644 --- a/src/jsonid/jsonid.py +++ b/src/jsonid/jsonid.py @@ -4,6 +4,7 @@ import argparse import asyncio +import copy import logging import signal import sys @@ -16,13 +17,30 @@ import helpers import lookup import registry + import registry_data import local except ModuleNotFoundError: try: - from src.jsonid import export, file_processing, helpers, local, lookup, registry + from src.jsonid import ( + export, + file_processing, + helpers, + local, + lookup, + registry, + registry_data, + ) except ModuleNotFoundError: - from jsonid import export, file_processing, helpers, local, lookup, registry + from jsonid import ( + export, + file_processing, + helpers, + local, + lookup, + registry, + registry_data, + ) logger = None @@ -295,11 +313,12 @@ def main() -> None: # Primary application functions. if args.registry: - local.load_and_parse_local_registry(path=args.registry) if args.localonly: - raise NotImplementedError("todo...") - - return + reg_data = local.load_and_parse_local_registry(path=args.registry) + else: + reg_data = local.load_and_parse_local_registry(path=args.registry) + if not args.registry: + reg_data = copy.deepcopy(registry_data.registry()) if args.pronom: export.export_pronom() sys.exit() @@ -355,6 +374,8 @@ def signal_handler(*args): # pylint: disable=W0613 strategy=strategy, binary=args.binary, agentout=args.agentout, + # TODO: registry data here? or just feed the local config through? + reg_data=reg_data, ) ) diff --git a/src/jsonid/local.py b/src/jsonid/local.py index 585e25b..fb54db5 100644 --- a/src/jsonid/local.py +++ b/src/jsonid/local.py @@ -1,5 +1,6 @@ """Functions supporting local registry use""" +import copy import logging import pathlib import tomllib as toml @@ -46,7 +47,7 @@ def load_and_parse_local_registry(path: str): """ -def load_local_registry(registry: pathlib.Path): +def load_local_registry(registry: pathlib.Path, only_local: bool = False): """Load the local registry and return it as a data structure to the caller. """ @@ -54,13 +55,13 @@ def load_local_registry(registry: pathlib.Path): with registry.open() as data: local_registry_data = data.read() - local_reg = toml.loads(local_registry_data) + local_reg_config = toml.loads(local_registry_data) - logger.debug("local registry length: %d", len(local_reg["entries"])) + logger.debug("local registry length: %d", len(local_reg_config["entries"])) - reg = registry_data.registry() + local_reg = [] - for item in local_reg["entries"]: + for item in local_reg_config["entries"]: # TODO: cleanup, ensure keys are capitalized. m = [] @@ -73,8 +74,8 @@ def load_local_registry(registry: pathlib.Path): # TODO: variable naming. a = registry_class.RegistryEntry( identifier=item["identifier"], - name=[{"@en": "TODO"}], - description=[{"@en": "TODO"}], + name=[{"@en": item.get("name")}], + description=[{"@en": item.get("description")}], markers=m, ) @@ -82,25 +83,11 @@ def load_local_registry(registry: pathlib.Path): print(a.markers) print("---") # if local... - reg.append(a) + local_reg.append(a) - # print(reg) + if only_local: + return local_reg - """ - [[entries]] - - name = "doctype1" - identifier = "local0001" - localref = "http://example.com/repository/ID - - [[entries.markers]] - - key = "key1" - is = "value1" - - {'name': 'doctype1', 'identifier': 'local0001', 'markers': [{'key': 'key1', 'is': 'value1'}]} - - - """ + reg = copy.deepcopy(registry_data.registry()) - assert False + return reg + local_reg diff --git a/src/jsonid/registry.py b/src/jsonid/registry.py index 0d24974..395378c 100644 --- a/src/jsonid/registry.py +++ b/src/jsonid/registry.py @@ -1,4 +1,4 @@ -"""JSON registry processor. """ +"""JSON registry processor.""" import copy import json @@ -20,7 +20,7 @@ registry_matchers, ) except ModuleNotFoundError: - from jsonid import analysis, registry_class, registry_data, registry_matchers + from jsonid import analysis, registry_class, registry_matchers logger = logging.getLogger(__name__) @@ -306,7 +306,7 @@ def build_identifier( return match_obj -def matcher(base_obj: BaseCharacteristics) -> list: +def matcher(base_obj: BaseCharacteristics, reg_data: list) -> list: """Matcher for registry objects.""" logger.debug("type: '%s'", type(base_obj.data)) if isinstance(base_obj.data, str): @@ -315,9 +315,15 @@ def matcher(base_obj: BaseCharacteristics) -> list: except json.decoder.JSONDecodeError as err: logger.error("unprocessable data: %s", err) return [] - reg = registry_data.registry() + + # TODO: load registry outside of this function and supply it as an + # arg to enable local registry functions? + # TODO: load registry outside of this function and supply it as an + # arg to enable local registry functions? + # TODO: load registry outside of this function and supply it as an + # arg to enable local registry functions? matches = [] - for idx, registry_entry in enumerate(reg): + for idx, registry_entry in enumerate(reg_data): try: logger.debug("processing registry entry: %s", idx) match = process_markers(registry_entry, base_obj.data) diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py index 82559b2..825d346 100644 --- a/tests/test_local_registry.py +++ b/tests/test_local_registry.py @@ -1,12 +1,14 @@ """Test functions associated with the local registry.""" +import copy +import tomllib from typing import Final -from src.jsonid import local +import pytest -registry: Final[ - str -] = """ +from src.jsonid import local, registry_data, registry, file_processing + +local_registry: Final[str] = """ [[entries]] name = "doctype1" @@ -26,6 +28,7 @@ name = "doctype2" identifier = "local0002" +description = "description TODO" [[entries.markers]] @@ -34,11 +37,50 @@ """ +test_file = """ +{ + "key2": "value2" +} +""" -def test_load_local(tmp_path): - """Ensure loading the local registry works as anticipated.""" - - a = tmp_path / "registry_path" - a.write_text(registry) - local.load_local_registry(a) +def test_load_local(tmp_path): + """Ensure loading the local registry and combining it with the + inbuilt registry works as anticipated.""" + + reg = copy.deepcopy(registry_data.registry()) + local_reg_path = tmp_path / "registry_path" + local_reg_path.write_text(local_registry) + local_reg_conf = tomllib.loads(local_registry) + assert len(reg) != len(local_reg_conf["entries"]) + local_reg = local.load_local_registry(local_reg_path) + assert len(local_reg) == len(reg) + len(local_reg_conf["entries"]) + + +@pytest.mark.asyncio +async def test_load_local_only(tmp_path): + """Ensure that just loading the local registry works as + anticipated. + """ + + local_reg_path = tmp_path / "registry_path" + local_reg_path.write_text(local_registry) + local_reg_conf = tomllib.loads(local_registry) + local_reg = local.load_local_registry(local_reg_path, only_local=True) + assert len(local_reg) == len(local_reg_conf["entries"]) + + test_file_path = tmp_path / "test_file.json" + test_file_path.write_text(test_file) + + base_obj = await file_processing.identify_plaintext_bytestream( + path=test_file_path, + strategy=["JSON"], + ) + + id_ = registry.matcher( + base_obj=base_obj, + reg_data=local_reg, + ) + + print(id_) + assert False From 2badcf9e51fe6ee3a07b245e25e16b326d5788a0 Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Mon, 7 Sep 2026 23:58:42 +0200 Subject: [PATCH 10/10] WIP; progress --- tests/test_local_registry.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py index 825d346..c88d058 100644 --- a/tests/test_local_registry.py +++ b/tests/test_local_registry.py @@ -83,4 +83,9 @@ async def test_load_local_only(tmp_path): ) print(id_) + + assert len(id_) == 1 + assert id_[0].identifier == "local0002" + assert id_[0].name[0]["@en"] == "doctype2" + assert False