diff --git a/.gitignore b/.gitignore index 6071354..8e77aea 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,7 @@ jsonid-integration-files/ # Secreta token.pypi jsonid_pronom.xml +*.csv +*.json +*.diff +token-pypi diff --git a/README.md b/README.md index 1a23ed0..cacd77d 100644 --- a/README.md +++ b/README.md @@ -405,9 +405,39 @@ within the `sops` object/value. ### Local rules -The plan is to allow local rules to be run alongside the global ruleset. I -expect this will be a bit further down the line when the ruleset and -metaddata is more stabilised. +You can define local rules in a local registry object. Local registries are +defined in TOML, and look as follows: + +```toml +[[entries]] + +name = "doctype1" +identifier = "local0001" +localref = "http://example.com/doctype/spec/ID" + +[[entries.markers]] + +key = "key1" +is = "value1" + +[[entries]] + +name = "doctype2" +identifier = "local0002" +localref = "http://example.com/doctype/spec/ID + +[[entries.markers]] + +key = "key2" +is = "value2" +``` + +Markers follow the same pattern as the standard registry. + +#### Local only + +Use the `--localonly` flag to use _just_ your custom markers in your format +identification workflow. ## PRONOM diff --git a/local/registry.toml b/local/registry.toml new file mode 100644 index 0000000..a0c36eb --- /dev/null +++ b/local/registry.toml @@ -0,0 +1,25 @@ +[[entries]] + +name = "doctype1" +identifier = "local0001" +localref = "http://example.com/repository/ID" + +[[entries.markers]] + +key = "key1" +is = "value1" + +[[entries.markers]] + +key = "key2" +is = "value2" + +[[entries]] + +name = "doctype2" +identifier = "local0002" + +[[entries.markers]] + +key = "key2" +is = "value2" diff --git a/src/jsonid/file_processing.py b/src/jsonid/file_processing.py index fa5927e..ff888d4 100644 --- a/src/jsonid/file_processing.py +++ b/src/jsonid/file_processing.py @@ -164,6 +164,9 @@ async def analyse_json(paths: list[str], strategy: list): """Analyse a JSON object.""" analysis_res = [] for path in paths: + if os.path.islink(path): + logger.debug(f"'{path}' is a symlink") + continue if os.path.getsize(path) == 0: logger.debug("%s is an empty file", path) continue @@ -200,8 +203,13 @@ async def process_result( base_obj: registry.BaseCharacteristics, padding: int, agentout: bool, + reg_data: list, ): - """Process something JSON/YAML/TOML""" + """Process something JSON/YAML/TOML. + + TODO: ... + + """ results = [] # NB. these switch-like ifs might not be needed in the fullness # of time. It depends if we need to do any custom processing of @@ -225,21 +233,23 @@ async def process_result( return # If we don't exit early and we try and identify the file... we then # create a new class object with an identification... - if base_obj.doctype == registry.DOCTYPE_JSON: - results = registry.matcher(base_obj) - if base_obj.doctype == registry.DOCTYPE_JSONL: - results = registry.matcher(base_obj) - if base_obj.doctype == registry.DOCTYPE_YAML: - results = registry.matcher(base_obj) - if base_obj.doctype == registry.DOCTYPE_TOML: - results = registry.matcher(base_obj) + if base_obj.doctype not in ( + registry.DOCTYPE_JSON, + registry.DOCTYPE_JSONL, + registry.DOCTYPE_YAML, + registry.DOCTYPE_TOML, + ): + return + results = registry.matcher( + base_obj=base_obj, + reg_data=reg_data, + ) output.output_results( path=path, results=results, padding=padding, agentout=agentout, ) - return def _get_padding(paths: list): @@ -255,10 +265,18 @@ def _get_padding(paths: list): return padding -async def identify_json(paths: list[str], strategy: list, binary: bool, agentout: bool): - """Identify objects.""" +async def identify_json( + paths: list[str], strategy: list, binary: bool, agentout: bool, reg_data: list +): + """Identify objects. + + TODO: ,,, + """ padding = _get_padding(paths=paths) for _, path in enumerate(paths): + if os.path.islink(path): + logger.debug(f"'{path}' is a symlink") + continue if os.path.getsize(path) == 0: logger.debug("%s is an empty file", path) base_obj = registry.BaseCharacteristics(empty=True) @@ -268,6 +286,7 @@ async def identify_json(paths: list[str], strategy: list, binary: bool, agentout base_obj=base_obj, padding=padding, agentout=agentout, + reg_data=reg_data, ) continue base_obj = await identify_plaintext_bytestream( @@ -283,6 +302,7 @@ async def identify_json(paths: list[str], strategy: list, binary: bool, agentout base_obj=base_obj, padding=padding, agentout=agentout, + reg_data=reg_data, ) continue logger.debug("processing: %s (%s)", path, base_obj.doctype) @@ -291,6 +311,7 @@ async def identify_json(paths: list[str], strategy: list, binary: bool, agentout base_obj=base_obj, padding=padding, agentout=agentout, + reg_data=reg_data, ) @@ -443,7 +464,9 @@ async def process_glob(glob_path: str): return paths -async def process_data(path: str, strategy: list, binary: bool, agentout: bool): +async def process_data( + path: str, strategy: list, binary: bool, agentout: bool, reg_data: list +): """Process all objects at a given path.""" logger.debug("processing: %s", path) if "*" in path: @@ -453,6 +476,7 @@ async def process_data(path: str, strategy: list, binary: bool, agentout: bool): strategy=strategy, binary=binary, agentout=agentout, + reg_data=reg_data, ) sys.exit(0) if not os.path.exists(path): @@ -464,6 +488,7 @@ async def process_data(path: str, strategy: list, binary: bool, agentout: bool): strategy=strategy, binary=binary, agentout=agentout, + reg_data=reg_data, ) sys.exit(0) paths = await create_manifest(path) @@ -475,6 +500,7 @@ async def process_data(path: str, strategy: list, binary: bool, agentout: bool): strategy=strategy, binary=binary, agentout=agentout, + reg_data=reg_data, ) diff --git a/src/jsonid/jsonid.py b/src/jsonid/jsonid.py index 9d3379e..eb64a2d 100644 --- a/src/jsonid/jsonid.py +++ b/src/jsonid/jsonid.py @@ -4,6 +4,7 @@ import argparse import asyncio +import copy import logging import signal import sys @@ -16,11 +17,30 @@ import helpers import lookup import registry + import registry_data + + import local except ModuleNotFoundError: try: - from src.jsonid import export, file_processing, helpers, lookup, registry + from src.jsonid import ( + export, + file_processing, + helpers, + local, + lookup, + registry, + registry_data, + ) except ModuleNotFoundError: - from jsonid import export, file_processing, helpers, lookup, registry + from jsonid import ( + export, + file_processing, + helpers, + local, + lookup, + registry, + registry_data, + ) logger = None @@ -216,9 +236,18 @@ def main() -> None: ) parser.add_argument( "--registry", + "--local", help="path to a custom registry to lead into memory replacing the default", required=False, ) + parser.add_argument( + "--localonly", + "--lonly", + "--lonely", + help="if a local registry is specified, use this and this only", + required=False, + action="store_true", + ) # NB. consider output to stdout once the feature is more stable. parser.add_argument( "--pronom", @@ -284,7 +313,12 @@ def main() -> None: # Primary application functions. if args.registry: - raise NotImplementedError("custom registry is not yet available") + if args.localonly: + reg_data = local.load_and_parse_local_registry(path=args.registry) + else: + reg_data = local.load_and_parse_local_registry(path=args.registry) + if not args.registry: + reg_data = copy.deepcopy(registry_data.registry()) if args.pronom: export.export_pronom() sys.exit() @@ -305,6 +339,8 @@ def main() -> None: logger.info("ok") sys.exit() if args.html: + if args.registry: + raise NotImplementedError("local registry output is not yet supported") helpers.html() sys.exit() if not strategy: @@ -338,6 +374,8 @@ def signal_handler(*args): # pylint: disable=W0613 strategy=strategy, binary=args.binary, agentout=args.agentout, + # TODO: registry data here? or just feed the local config through? + reg_data=reg_data, ) ) diff --git a/src/jsonid/local.py b/src/jsonid/local.py new file mode 100644 index 0000000..fb54db5 --- /dev/null +++ b/src/jsonid/local.py @@ -0,0 +1,93 @@ +"""Functions supporting local registry use""" + +import copy +import logging +import pathlib +import tomllib as toml + +try: + import registry_class + import registry_data +except ModuleNotFoundError: + try: + from src.jsonid import registry_class, registry_data + except ModuleNotFoundError: + from jsonid import registry_class, registry_data + +logger = logging.getLogger(__name__) + + +class LocalRegistryException(Exception): + """Exception to raise if something goes wrong with the local + registry. + """ + + +def load_and_parse_local_registry(path: str): + """Read the data use the data.""" + + registry = pathlib.Path(path) + if not registry.exists(): + raise LocalRegistryException("registry path not found") + + load_local_registry(registry) + + +""" + registry_class.RegistryEntry( + identifier="jrid:0001", + name=[{"@en": "JavaScript Package Lock"}], + description=[{"@en": "describes an exact Node (NPM) module dependency tree"}], + markers=[ + {"KEY": "name", "EXISTS": None}, + {"KEY": "lockfileVersion", "EXISTS": None}, + {"KEY": "packages", "EXISTS": None}, + ], + ), +""" + + +def load_local_registry(registry: pathlib.Path, only_local: bool = False): + """Load the local registry and return it as a data structure + to the caller. + """ + + with registry.open() as data: + local_registry_data = data.read() + + local_reg_config = toml.loads(local_registry_data) + + logger.debug("local registry length: %d", len(local_reg_config["entries"])) + + local_reg = [] + + for item in local_reg_config["entries"]: + + # TODO: cleanup, ensure keys are capitalized. + m = [] + for i in item["markers"]: + d = {} + for k, v in tuple(i.items()): + d.update({k.upper(): v}) + m.append(d) + + # TODO: variable naming. + a = registry_class.RegistryEntry( + identifier=item["identifier"], + name=[{"@en": item.get("name")}], + description=[{"@en": item.get("description")}], + markers=m, + ) + + print(a) + print(a.markers) + print("---") + # if local... + local_reg.append(a) + + if only_local: + return local_reg + + reg = copy.deepcopy(registry_data.registry()) + + return reg + local_reg diff --git a/src/jsonid/registry.py b/src/jsonid/registry.py index 0d24974..395378c 100644 --- a/src/jsonid/registry.py +++ b/src/jsonid/registry.py @@ -1,4 +1,4 @@ -"""JSON registry processor. """ +"""JSON registry processor.""" import copy import json @@ -20,7 +20,7 @@ registry_matchers, ) except ModuleNotFoundError: - from jsonid import analysis, registry_class, registry_data, registry_matchers + from jsonid import analysis, registry_class, registry_matchers logger = logging.getLogger(__name__) @@ -306,7 +306,7 @@ def build_identifier( return match_obj -def matcher(base_obj: BaseCharacteristics) -> list: +def matcher(base_obj: BaseCharacteristics, reg_data: list) -> list: """Matcher for registry objects.""" logger.debug("type: '%s'", type(base_obj.data)) if isinstance(base_obj.data, str): @@ -315,9 +315,15 @@ def matcher(base_obj: BaseCharacteristics) -> list: except json.decoder.JSONDecodeError as err: logger.error("unprocessable data: %s", err) return [] - reg = registry_data.registry() + + # TODO: load registry outside of this function and supply it as an + # arg to enable local registry functions? + # TODO: load registry outside of this function and supply it as an + # arg to enable local registry functions? + # TODO: load registry outside of this function and supply it as an + # arg to enable local registry functions? matches = [] - for idx, registry_entry in enumerate(reg): + for idx, registry_entry in enumerate(reg_data): try: logger.debug("processing registry entry: %s", idx) match = process_markers(registry_entry, base_obj.data) diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py new file mode 100644 index 0000000..c88d058 --- /dev/null +++ b/tests/test_local_registry.py @@ -0,0 +1,91 @@ +"""Test functions associated with the local registry.""" + +import copy +import tomllib +from typing import Final + +import pytest + +from src.jsonid import local, registry_data, registry, file_processing + +local_registry: Final[str] = """ +[[entries]] + +name = "doctype1" +identifier = "local0001" + +[[entries.markers]] + +key = "key1" +is = "value1" + +[[entries.markers]] + +key = "key2" +is = "value2" + +[[entries]] + +name = "doctype2" +identifier = "local0002" +description = "description TODO" + +[[entries.markers]] + +key = "key2" +is = "value2" + +""" + +test_file = """ +{ + "key2": "value2" +} +""" + + +def test_load_local(tmp_path): + """Ensure loading the local registry and combining it with the + inbuilt registry works as anticipated.""" + + reg = copy.deepcopy(registry_data.registry()) + local_reg_path = tmp_path / "registry_path" + local_reg_path.write_text(local_registry) + local_reg_conf = tomllib.loads(local_registry) + assert len(reg) != len(local_reg_conf["entries"]) + local_reg = local.load_local_registry(local_reg_path) + assert len(local_reg) == len(reg) + len(local_reg_conf["entries"]) + + +@pytest.mark.asyncio +async def test_load_local_only(tmp_path): + """Ensure that just loading the local registry works as + anticipated. + """ + + local_reg_path = tmp_path / "registry_path" + local_reg_path.write_text(local_registry) + local_reg_conf = tomllib.loads(local_registry) + local_reg = local.load_local_registry(local_reg_path, only_local=True) + assert len(local_reg) == len(local_reg_conf["entries"]) + + test_file_path = tmp_path / "test_file.json" + test_file_path.write_text(test_file) + + base_obj = await file_processing.identify_plaintext_bytestream( + path=test_file_path, + strategy=["JSON"], + ) + + id_ = registry.matcher( + base_obj=base_obj, + reg_data=local_reg, + ) + + print(id_) + + assert len(id_) == 1 + assert id_[0].identifier == "local0002" + assert id_[0].name[0]["@en"] == "doctype2" + + assert False