From 0f3064f39f878e1071f54b69e9d24fc5a8df3c29 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 23:07:19 +0000 Subject: [PATCH 1/6] Test that the JSON Builder writes a list of records --- .../tests/test_jsonbundler_records.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py new file mode 100644 index 00000000..8bb44eba --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py @@ -0,0 +1,61 @@ + +""" +A Property whose value is a list of Element Data is written as a list of +objects, each following the null rules of the JSON Builder specification, and +the other Properties are written beside it. +""" + +import unittest + +from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class ProfilesElement(FlowElement): + + def __init__(self): + super(ProfilesElement, self).__init__() + self.datakey = "hardware" + self.properties = { + "profiles": {"name": "Profiles", "type": "array"}, + "vendor": {"name": "Vendor", "type": "string"}} + + def process_internal(self, flowdata): + records = [ + {"hardwarename": AspectPropertyValue(value="Phone"), + "hardwarevendor": AspectPropertyValue(no_value_message="No data"), + "hardwaremodel": None}, + {"hardwarename": AspectPropertyValue(value="Tablet")}] + flowdata.set_element_data(ElementDataDictionary( + self, {"profiles": records, + "vendor": AspectPropertyValue(value="Acme")})) + + +class JsonBuilderRecordTests(unittest.TestCase): + + def process(self): + pipeline = PipelineBuilder({}).add(ProfilesElement()).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.process() + return flowdata.jsonbundler.json["hardware"] + + def test_each_record_is_an_object(self): + self.assertEqual( + [{"hardwarename": "Phone", + "hardwarevendor": None, + "hardwarevendornullreason": "No data"}, + {"hardwarename": "Tablet"}], + self.process()["profiles"]) + + def test_a_null_that_was_not_assigned_no_value_has_no_entry(self): + self.assertNotIn("hardwaremodel", self.process()["profiles"][0]) + + def test_the_other_properties_are_still_written(self): + self.assertEqual("Acme", self.process()["vendor"]) + + +if __name__ == "__main__": + unittest.main() From 130228fdcbc7a81ce943d5486925a62ac5763ec9 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 23:07:46 +0000 Subject: [PATCH 2/6] Write a list of records in the JSON Builder A Property whose value is a list of Element Data is written as a list of objects. A Property with no value is null beside its nullreason, and a null that was not assigned no value has no entry. The list branch no longer replaces the output with its own list, and an empty list is written as an empty list. --- .../src/fiftyone_pipeline_core/jsonbundler.py | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index 9f2a8ca7..8e32ad1e 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -47,6 +47,36 @@ def __init__(self): self.property_cache = {} + @staticmethod + def record_json(record): + + """! + A record, being the Properties of one Element Data in a list, as an + object. A Property that has been assigned no value is null with its + nullreason beside it, and a Property that is null without one has no + entry. + @type record: dict + @param record: Property name to value or aspect property value + @rtype: dict + @return: The record as it is written to JSON + """ + + written = {} + + for key, value in record.items(): + key = str(key).lower() + + if hasattr(value, "has_value"): + if value.has_value(): + written[key] = value.value() + else: + written[key] = None + written[key + "nullreason"] = value.no_value_message() + elif value is not None: + written[key] = value + + return written + def process_internal(self, flowdata): """! @@ -150,19 +180,25 @@ def process_internal(self, flowdata): value = None null_reason = value_container.no_value_message() - # Check if list of aspect property values + # A list of records, such as the profiles of a hardware + # profile lookup, or of aspect property values - elif isinstance(value_container, list) and isinstance(value_container[0], object): + elif isinstance(value_container, list): - output = [] + items = [] for item in value_container: - if item.has_value(): - output.append(item.value()) + if isinstance(item, dict): + items.append(self.record_json(item)) + elif hasattr(item, "has_value"): + if item.has_value(): + items.append(item.value()) + else: + null_reason = item.no_value_message() else: - null_reason = item.no_value_message + items.append(item) - value = output + value = items else: From 07ad8b9b33f830295d24bbb1708006fed38bdffe Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 10:26:38 +0000 Subject: [PATCH 3/6] Test that a top-level null not assigned no value has no entry in the JSON Builder --- .../tests/test_jsonbundler_records.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py index 8bb44eba..924edbcd 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py @@ -20,7 +20,9 @@ def __init__(self): self.datakey = "hardware" self.properties = { "profiles": {"name": "Profiles", "type": "array"}, - "vendor": {"name": "Vendor", "type": "string"}} + "vendor": {"name": "Vendor", "type": "string"}, + "model": {"name": "Model", "type": "string"}, + "colour": {"name": "Colour", "type": "string"}} def process_internal(self, flowdata): records = [ @@ -30,7 +32,10 @@ def process_internal(self, flowdata): {"hardwarename": AspectPropertyValue(value="Tablet")}] flowdata.set_element_data(ElementDataDictionary( self, {"profiles": records, - "vendor": AspectPropertyValue(value="Acme")})) + "vendor": AspectPropertyValue(value="Acme"), + "model": None, + "colour": AspectPropertyValue( + no_value_message="No colour")})) class JsonBuilderRecordTests(unittest.TestCase): @@ -56,6 +61,16 @@ def test_a_null_that_was_not_assigned_no_value_has_no_entry(self): def test_the_other_properties_are_still_written(self): self.assertEqual("Acme", self.process()["vendor"]) + def test_a_top_level_null_not_assigned_no_value_has_no_entry(self): + hardware = self.process() + self.assertNotIn("model", hardware) + self.assertNotIn("modelnullreason", hardware) + + def test_a_top_level_property_assigned_no_value_is_null(self): + hardware = self.process() + self.assertIsNone(hardware["colour"]) + self.assertEqual("No colour", hardware["colournullreason"]) + if __name__ == "__main__": unittest.main() From c448593c62d2d791ea5878412bc6270377922af0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 10:26:40 +0000 Subject: [PATCH 4/6] Leave a top-level null not assigned no value out of the JSON, as the specification says --- .../src/fiftyone_pipeline_core/jsonbundler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index 8e32ad1e..9d1ddf01 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -150,6 +150,7 @@ def process_internal(self, flowdata): for propertykey, propertymeta in properties.items(): value = None null_reason = "Unknown" + no_value = False # Check if property has delayed execution and set in JSON if yes @@ -178,6 +179,7 @@ def process_internal(self, flowdata): value = value_container.value() else: value = None + no_value = True null_reason = value_container.no_value_message() # A list of records, such as the profiles of a hardware @@ -193,8 +195,6 @@ def process_internal(self, flowdata): elif hasattr(item, "has_value"): if item.has_value(): items.append(item.value()) - else: - null_reason = item.no_value_message() else: items.append(item) @@ -209,6 +209,11 @@ def process_internal(self, flowdata): # Catching missing property exceptions and other errors continue + + # A null that was not assigned 'no value', which an element + # that is not an engine can give, has no entry of its own. + if value is None and not no_value: + continue output[flow_element.datakey.lower()][propertykey.lower()] = value if value is None: From efb99d9dfa66bd7f57b40ffb61a4b57f294eebc1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:29:55 +0000 Subject: [PATCH 5/6] Test that a null JavaScript Property is still listed and a no value with no message has a text reason --- .../tests/test_jsonbundler_records.py | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py index 924edbcd..e6ff5cf0 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py @@ -22,37 +22,46 @@ def __init__(self): "profiles": {"name": "Profiles", "type": "array"}, "vendor": {"name": "Vendor", "type": "string"}, "model": {"name": "Model", "type": "string"}, - "colour": {"name": "Colour", "type": "string"}} + "colour": {"name": "Colour", "type": "string"}, + "script": {"name": "Script", "type": "javascript"}, + "size": {"name": "Size", "type": "string"}} def process_internal(self, flowdata): records = [ {"hardwarename": AspectPropertyValue(value="Phone"), "hardwarevendor": AspectPropertyValue(no_value_message="No data"), "hardwaremodel": None}, - {"hardwarename": AspectPropertyValue(value="Tablet")}] + {"hardwarename": AspectPropertyValue(value="Tablet")}, + {"hardwarename": AspectPropertyValue()}] flowdata.set_element_data(ElementDataDictionary( self, {"profiles": records, "vendor": AspectPropertyValue(value="Acme"), "model": None, "colour": AspectPropertyValue( - no_value_message="No colour")})) + no_value_message="No colour"), + "script": None, + "size": AspectPropertyValue()})) class JsonBuilderRecordTests(unittest.TestCase): - def process(self): + def process_json(self): pipeline = PipelineBuilder({}).add(ProfilesElement()).build() self.addCleanup(pipeline.close) flowdata = pipeline.create_flowdata() flowdata.process() - return flowdata.jsonbundler.json["hardware"] + return flowdata.jsonbundler.json + + def process(self): + return self.process_json()["hardware"] def test_each_record_is_an_object(self): self.assertEqual( [{"hardwarename": "Phone", "hardwarevendor": None, "hardwarevendornullreason": "No data"}, - {"hardwarename": "Tablet"}], + {"hardwarename": "Tablet"}, + {"hardwarename": None, "hardwarenamenullreason": "Unknown"}], self.process()["profiles"]) def test_a_null_that_was_not_assigned_no_value_has_no_entry(self): @@ -66,6 +75,16 @@ def test_a_top_level_null_not_assigned_no_value_has_no_entry(self): self.assertNotIn("model", hardware) self.assertNotIn("modelnullreason", hardware) + def test_a_null_javascript_property_is_still_listed(self): + json = self.process_json() + self.assertNotIn("script", json["hardware"]) + self.assertIn("hardware.script", json["javascriptProperties"]) + + def test_no_value_with_no_message_has_a_text_reason(self): + hardware = self.process() + self.assertIsNone(hardware["size"]) + self.assertEqual("Unknown", hardware["sizenullreason"]) + def test_a_top_level_property_assigned_no_value_is_null(self): hardware = self.process() self.assertIsNone(hardware["colour"]) From 8226252122cc1f33cd108b2b111a8ce6b82bb079 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:30:39 +0000 Subject: [PATCH 6/6] List a null JavaScript Property, and give a no value with no message the reason Unknown --- .../src/fiftyone_pipeline_core/jsonbundler.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index e791614f..fa300a7a 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -81,7 +81,8 @@ def record_json(record): written[key] = value.value() else: written[key] = None - written[key + "nullreason"] = value.no_value_message() + written[key + "nullreason"] = ( + value.no_value_message() or "Unknown") elif value is not None: written[key] = value @@ -190,7 +191,9 @@ def process_internal(self, flowdata): else: value = None no_value = True - null_reason = value_container.no_value_message() + null_reason = ( + value_container.no_value_message() + or null_reason) # A list of records, such as the profiles of a hardware # profile lookup, or of aspect property values @@ -221,13 +224,12 @@ def process_internal(self, flowdata): continue # A null that was not assigned 'no value', which an element - # that is not an engine can give, has no entry of its own. - if value is None and not no_value: - continue - - output[flow_element.datakey.lower()][propertykey.lower()] = value - if value is None: - output[flow_element.datakey.lower()][propertykey.lower() + "nullreason"] = null_reason + # that is not an engine can give, has no entry of its own. A + # JavaScript Property is still listed below. + if value is not None or no_value: + output[flow_element.datakey.lower()][propertykey.lower()] = value + if value is None: + output[flow_element.datakey.lower()][propertykey.lower() + "nullreason"] = null_reason sequence = flowdata.evidence.get("query.sequence")