From eb2a2e5e2fd9ebe81de1323e6645d1b503983a32 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 17:50:54 +0000 Subject: [PATCH 01/11] Test the usage sharing record attributes --- .../tests/test_share_usage_record.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py new file mode 100644 index 00000000..269c56b7 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py @@ -0,0 +1,48 @@ + +""" +How the usage sharing element writes an Evidence value that was changed, as +the Processing section of the usage sharing element specification says. +""" + +import unittest + +from fiftyone_pipeline_engines_fiftyone.share_usage import ( + SHARE_USAGE_MAX_EVIDENCE_LENGTH, ShareUsage, ShareUsageData) + + +class ShareUsageRecordTests(unittest.TestCase): + + def record(self, key, value): + element = ShareUsage(requested_package_size=1000) + self.addCleanup(element.close) + element.constant_xml = "test" + data = ShareUsageData() + data.session_id = "session" + data.sequence = 1 + data.try_add(key, value) + element.add_to_share_usage(data) + return element.share_data[0] + + def testUnchangedValueHasNoAttributes(self): + xml = self.record("header.user-agent", "Mozilla/5.0") + self.assertIn('
Mozilla/5.0
', xml) + + def testInvalidCharacterIsMarkedEscaped(self): + xml = self.record("header.user-agent", "a\x01b") + self.assertIn(' escaped="true"', xml) + self.assertNotIn("replaced", xml) + + def testLongValueIsMarkedTruncated(self): + xml = self.record( + "header.user-agent", "a" * (SHARE_USAGE_MAX_EVIDENCE_LENGTH + 1)) + self.assertIn(' truncated="true"', xml) + self.assertNotIn("escaped", xml) + + def testChangedValueIsWrittenWithoutCdataOrBadSchema(self): + xml = self.record("header.user-agent", "a\x01b") + self.assertNotIn("CDATA", xml) + self.assertNotIn("BadSchema", xml) + + +if __name__ == "__main__": + unittest.main() From b34c488b3535a8abbc1019ef8c72ceda8e70a54a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 17:52:47 +0000 Subject: [PATCH 02/11] Mark a value with invalid characters as escaped in the usage sharing record --- .../src/fiftyone_pipeline_engines_fiftyone/share_usage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 210dfb0a..2315b8b2 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -285,7 +285,7 @@ def add_to_share_usage(self, data): xml += f"<{entry_key}" # Write any attributes if replaced_string.replaced: - xml += ' replaced="true"' + xml += ' escaped="true"' if replaced_string.truncated: xml += ' truncated="true"' # End the start element From 46579e11060c1807874a92323890d23cc8e934d3 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 17:55:20 +0000 Subject: [PATCH 03/11] Expect escaped in the usage sharing invalid character test --- pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 4f1bea40..f68efad0 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -317,7 +317,7 @@ def test_invalid_char(self): self.assertEqual(len(app.received), 1) self.assertNotIn("ƌ", app.received[0]) self.assertIn(replacementChar, app.received[0]) - self.assertIn(f"
1{replacementChar}2
", app.received[0]) + self.assertIn(f"
1{replacementChar}2
", app.received[0]) def test_truncated_value(self): """! From 6dcef57e9ea09dcaea9ff33dd5711ba53f3fc526 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:31:46 +0000 Subject: [PATCH 04/11] Test the usage sharing record escapes and limit --- .../tests/test_share_usage_record.py | 80 +++++++++++++++---- .../tests/test_usage.py | 13 ++- 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py index 269c56b7..5a12a113 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py @@ -1,7 +1,8 @@ """ How the usage sharing element writes an Evidence value that was changed, as -the Processing section of the usage sharing element specification says. +the Processing section of the usage sharing element specification says, with +the invalid character and length rules of specifications pull request 122. """ import unittest @@ -9,10 +10,12 @@ from fiftyone_pipeline_engines_fiftyone.share_usage import ( SHARE_USAGE_MAX_EVIDENCE_LENGTH, ShareUsage, ShareUsageData) +BS = chr(92) + class ShareUsageRecordTests(unittest.TestCase): - def record(self, key, value): + def record(self, value, key="header.user-agent"): element = ShareUsage(requested_package_size=1000) self.addCleanup(element.close) element.constant_xml = "test" @@ -23,25 +26,74 @@ def record(self, key, value): element.add_to_share_usage(data) return element.share_data[0] - def testUnchangedValueHasNoAttributes(self): - xml = self.record("header.user-agent", "Mozilla/5.0") + def test_limit_is_ten_thousand(self): + self.assertEqual(10000, SHARE_USAGE_MAX_EVIDENCE_LENGTH) + + def test_unchanged_value_has_no_attributes(self): + xml = self.record("Mozilla/5.0") self.assertIn('
Mozilla/5.0
', xml) - def testInvalidCharacterIsMarkedEscaped(self): - xml = self.record("header.user-agent", "a\x01b") - self.assertIn(' escaped="true"', xml) - self.assertNotIn("replaced", xml) + def test_invalid_character_is_written_as_hex_and_marked_escaped(self): + xml = self.record("a" + chr(0x1f) + "b") + self.assertIn( + '
a' + BS + 'x001fb
', + xml) - def testLongValueIsMarkedTruncated(self): - xml = self.record( - "header.user-agent", "a" * (SHARE_USAGE_MAX_EVIDENCE_LENGTH + 1)) - self.assertIn(' truncated="true"', xml) + def test_other_invalid_characters(self): + for code, text in ((0x0, "0000"), (0x8, "0008"), (0xb, "000b"), + (0xc, "000c"), (0xe, "000e"), (0xd800, "d800"), + (0xdfff, "dfff"), (0xfffe, "fffe"), + (0xffff, "ffff")): + xml = self.record("x" + chr(code) + "y") + self.assertIn("x" + BS + "x" + text + "y", xml, hex(code)) + self.assertIn(' escaped="true"', xml, hex(code)) + + def test_characters_xml_allows_are_kept(self): + value = "t" + chr(9) + chr(10) + chr(13) + chr(0x18c) + chr(0xfffd) + xml = self.record(value) + self.assertIn(">" + value + "<", xml) + self.assertNotIn("escaped", xml) + + def test_markup_characters_are_still_entities(self): + xml = self.record("a&bd\"e'") + self.assertIn(">a&b<c>d"e'<", xml) self.assertNotIn("escaped", xml) - def testChangedValueIsWrittenWithoutCdataOrBadSchema(self): - xml = self.record("header.user-agent", "a\x01b") + def test_value_of_the_limit_is_not_truncated(self): + xml = self.record("a" * 10000) + self.assertIn(">" + "a" * 10000 + "<", xml) + self.assertNotIn("truncated", xml) + + def test_longer_value_is_cut_and_marked_truncated(self): + xml = self.record("a" * 10001) + self.assertIn( + ' truncated="true">' + "a" * 10000 + "", xml) + + def test_length_is_counted_before_markup_characters_are_written(self): + xml = self.record("&" * 10000) + self.assertNotIn("truncated", xml) + + def test_character_outside_the_basic_plane_is_never_split(self): + emoji = chr(0x1f600) + xml = self.record("a" * 9999 + emoji + "tail") + self.assertIn( + ' truncated="true">' + "a" * 9999 + emoji + "", xml) + xml = self.record("a" * 9998 + emoji + emoji + "z") + self.assertIn( + ' truncated="true">' + "a" * 9998 + emoji + emoji + "", + xml) + self.assertNotIn("escaped", xml) + + def test_invalid_character_in_a_long_value_sets_both(self): + xml = self.record(chr(1) + "a" * 10000) + self.assertIn(' escaped="true"', xml) + self.assertIn(' truncated="true"', xml) + + def test_changed_value_is_written_without_cdata_or_bad_schema(self): + xml = self.record("a" + chr(1) + "b") self.assertNotIn("CDATA", xml) self.assertNotIn("BadSchema", xml) + self.assertNotIn("replaced", xml) if __name__ == "__main__": diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index f68efad0..5afaaee4 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -304,20 +304,17 @@ def test_invalid_char(self): requested_package_size=1, endpoint=testEndpoint) - replacementChar = chr(int("0xFFFD", 16)) - pipeline = (PipelineBuilder())\ .add(usage)\ .build() flowdata = pipeline.create_flowdata() - flowdata.evidence.add("header.user-agent", "1ƌ2") + flowdata.evidence.add("header.user-agent", "1" + chr(0x1f) + "2") flowdata.process() self.waitForUsageThreads() self.assertEqual(len(app.received), 1) - self.assertNotIn("ƌ", app.received[0]) - self.assertIn(replacementChar, app.received[0]) - self.assertIn(f"
1{replacementChar}2
", app.received[0]) + self.assertNotIn(chr(0x1f), app.received[0]) + self.assertIn('
1\\x001f2
', app.received[0]) def test_truncated_value(self): """! @@ -328,7 +325,7 @@ def test_truncated_value(self): requested_package_size=1, endpoint=testEndpoint) - ua = "X" * 1000 + ua = "X" * 10001 pipeline = (PipelineBuilder())\ .add(usage)\ @@ -341,7 +338,7 @@ def test_truncated_value(self): self.assertEqual(len(app.received), 1) self.assertIn("truncated=\"true\"", app.received[0]) self.assertNotIn(ua, app.received[0]) - self.assertIn(f'
{"X" * 512}
', app.received[0]) + self.assertIn(f'
{"X" * 10000}
', app.received[0]) def test_send_more_than_once(self): """! From ef68190c7e7abc290a4d6a9fce904911c3694b3d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:32:43 +0000 Subject: [PATCH 05/11] Write invalid characters as backslash x and four hex digits, and cut values at 10,000 characters A character XML 1.0 does not allow is written as its code in four lower-case hexadecimal digits after a backslash and x, and the element is marked escaped. A value is cut to 10,000 characters (code points) before the markup characters are written, and the element is marked truncated. --- .../share_usage.py | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 2315b8b2..3650c278 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -37,7 +37,7 @@ # The maximum length of a piece of evidence's value which can be # added to the usage data being sent. -SHARE_USAGE_MAX_EVIDENCE_LENGTH = 512 +SHARE_USAGE_MAX_EVIDENCE_LENGTH = 10000 SHARE_USAGE_VERSION = '1.1' @@ -284,7 +284,7 @@ def add_to_share_usage(self, data): else: xml += f"<{entry_key}" # Write any attributes - if replaced_string.replaced: + if replaced_string.escaped: xml += ' escaped="true"' if replaced_string.truncated: xml += ' truncated="true"' @@ -361,49 +361,51 @@ def get_data_from_evidence(self, flow_data): return data -# a set of valid XML character values (ignoring valid controls x09, x0a, x0d, x85) -VALID_XML_CHARS = [*range(int("0x20", 16), int("0x7F", 16))] -VALID_XML_CHARS.extend(range(int("0x40", 16), int("0x100", 16))) +# The characters that XML 1.0 section 2.2 allows, as ranges of code points. +VALID_XML_RANGES = ( + (0x9, 0xA), (0xD, 0xD), (0x20, 0xD7FF), (0xE000, 0xFFFD), + (0x10000, 0x10FFFF)) -def get_is_valid_char_map(): - maxChar = int('0x100', 16) - isValidChar = {} - for c in range(maxChar): - isValidChar[c] = c in VALID_XML_CHARS - return isValidChar +# The characters that are written as entities in element text. +XML_ENTITIES = { + '&': '&', + '"': '"', + "'": ''', + '<': '<', + '>': '>', +} -# an array describing whether a character value is valid -IS_VALID_XML_CHAR = get_is_valid_char_map() + +def is_valid_xml_char(code): + return any(low <= code <= high for low, high in VALID_XML_RANGES) class ReplacedString: """! - replace characters that cause problems in XML with the "Replacement character" + Makes an Evidence value fit in the usage sharing XML. The value is cut to + SHARE_USAGE_MAX_EVIDENCE_LENGTH characters (code points, so a character is + never split), a character XML 1.0 does not allow is written as a backslash, + "x" and its code in four lower-case hexadecimal digits, and the markup + characters are written as entities. `escaped` and `truncated` say what + changed. """ def __init__(self, text): self.result = "" - self.replaced = False + self.escaped = False self.truncated = False if text: - escapedText = text\ - .replace('&', '&')\ - .replace('"', '"')\ - .replace('\'', ''')\ - .replace('<', '<')\ - .replace('>', '>') - - length = min(len(escapedText), SHARE_USAGE_MAX_EVIDENCE_LENGTH) - self.truncated = len(escapedText) > SHARE_USAGE_MAX_EVIDENCE_LENGTH - self.result = "".join(map(self.map_char, list(escapedText[0:length]))) - + if not isinstance(text, str): + text = str(text) + self.truncated = len(text) > SHARE_USAGE_MAX_EVIDENCE_LENGTH + self.result = "".join( + map(self.map_char, text[0:SHARE_USAGE_MAX_EVIDENCE_LENGTH])) def map_char(self, char): - charV = ord(char) - if charV < len(IS_VALID_XML_CHAR) and IS_VALID_XML_CHAR[charV]: - return char - else: - self.replaced = True - return chr(int("0xFFFD", 16)) + code = ord(char) + if not is_valid_xml_char(code): + self.escaped = True + return "\\x%04x" % code + return XML_ENTITIES.get(char, char) class ShareUsageData: """! From b6ebdd603c04f66c5c823504ef30197103e62d51 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:43:19 +0000 Subject: [PATCH 06/11] Test that escapes do not count towards the usage sharing length limit --- .../tests/test_share_usage_record.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py index 5a12a113..30b2b0b7 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py @@ -73,6 +73,12 @@ def test_length_is_counted_before_markup_characters_are_written(self): xml = self.record("&" * 10000) self.assertNotIn("truncated", xml) + def test_escapes_do_not_count_towards_the_limit(self): + xml = self.record(chr(1) + "a" * 9999) + self.assertIn(' escaped="true"', xml) + self.assertNotIn("truncated", xml) + self.assertIn(chr(92) + "x0001" + "a" * 9999 + "<", xml) + def test_character_outside_the_basic_plane_is_never_split(self): emoji = chr(0x1f600) xml = self.record("a" * 9999 + emoji + "tail") From 3dc58564c029f12c92ca330314fb2c4da9fe35d1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 09:53:39 +0000 Subject: [PATCH 07/11] Test that the session id, client IP and names leave the usage record parseable, and the date and warning --- .../tests/test_share_usage_record.py | 65 +++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py index 30b2b0b7..d4c1b783 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py @@ -1,12 +1,16 @@ """ How the usage sharing element writes an Evidence value that was changed, as -the Processing section of the usage sharing element specification says, with -the invalid character and length rules of specifications pull request 122. +the Processing section of the usage sharing element specification says, +including its invalid character and length rules. """ +import re import unittest +import xml.etree.ElementTree as ElementTree +from fiftyone_pipeline_core.logger import Logger +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_engines_fiftyone.share_usage import ( SHARE_USAGE_MAX_EVIDENCE_LENGTH, ShareUsage, ShareUsageData) @@ -15,17 +19,70 @@ class ShareUsageRecordTests(unittest.TestCase): - def record(self, value, key="header.user-agent"): + def record(self, value, key="header.user-agent", session_id="session", + client_ip=""): element = ShareUsage(requested_package_size=1000) self.addCleanup(element.close) element.constant_xml = "test" data = ShareUsageData() - data.session_id = "session" + data.session_id = session_id + data.client_ip = client_ip data.sequence = 1 data.try_add(key, value) element.add_to_share_usage(data) return element.share_data[0] + def test_session_id_is_written_so_the_record_parses(self): + session_id = 'a"): + xml = self.record("a value", key=key) + device = ElementTree.fromstring(xml) + self.assertNotIn("a value", xml, key) + self.assertEqual("session", device.find("SessionId").text, key) + + def test_date_sent_is_utc_to_the_second(self): + device = ElementTree.fromstring(self.record("Mozilla/5.0")) + self.assertRegex( + device.find("DateSent").text, + re.compile(r"\A\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\Z")) + + def test_registered_to_too_many_pipelines_logs_a_warning(self): + logged = [] + + class Recorder(Logger): + def log_internal(self, level, log): + logged.append(level) + + element = ShareUsage(requested_package_size=1000) + self.addCleanup(element.close) + pipelines = [] + for _ in range(2): + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + builder.add_logger(Recorder("debug")) + pipelines.append(builder.build()) + element.pipelines = pipelines + self.assertEqual([], element.get_flow_elements()) + self.assertEqual(["warning", "warning"], logged) + def test_limit_is_ten_thousand(self): self.assertEqual(10000, SHARE_USAGE_MAX_EVIDENCE_LENGTH) From 2c78e296bd4477dac8f32e8eb69255a02e48b3ca Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 09:54:29 +0000 Subject: [PATCH 08/11] Write the session id, client IP and Name attribute through the record's escaping, leave out entries with no XML name, and give the date in UTC --- .../share_usage.py | 74 ++++++++++++------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 3650c278..176f208a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -25,6 +25,7 @@ import gzip import platform import datetime +import re from concurrent.futures import ThreadPoolExecutor from importlib.metadata import version @@ -203,7 +204,7 @@ def get_flow_elements(self): # make up the pipeline so a warning is logged # but otherwise, the system can continue as normal. self._log( - "warn", + "warning", "Share usage element registered " f"to {'too many' if len(self.pipelines) > 0 else 'no'}" " pipelines. Unable to send share usage information.") @@ -264,40 +265,30 @@ def add_to_share_usage(self, data): # --- write variable data # The SessionID used to track a series of requests - xml += f"{data.session_id}" + xml += record_element("SessionId", data.session_id) # The sequence number of the request in a series of requests. xml += f"{data.sequence}" # The client IP of the request - xml += f"{data.client_ip}" + xml += record_element("ClientIP", data.client_ip) - # The UTC date/time this entry was written - date = datetime.datetime.now().isoformat() + # The UTC date and time this entry was written, to the second. + date = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%S") xml += f"{date}" - # Write all other evidence data that has been included. + # Write all other evidence data that has been included. The prefix, + # or a key with no prefix, is the element name. An entry whose + # element name is not an XML name is left out, because it would make + # the whole batch unreadable. for category_key, category_value in data.evidence_data.items(): for entry_key, entry_value in category_value.items(): - replaced_string = ReplacedString(entry_value) - # Write start element if len(category_key) > 0: - xml += f"<{category_key} Name=\"{entry_key}\"" - else: - xml += f"<{entry_key}" - # Write any attributes - if replaced_string.escaped: - xml += ' escaped="true"' - if replaced_string.truncated: - xml += ' truncated="true"' - # End the start element - xml += ">" - # Write the value - xml += replaced_string.result - # Write end element - if len(category_key) > 0: - xml += f"" - else: - xml += f"" - + if XML_NAME.match(category_key): + xml += record_element( + category_key, entry_value, entry_key) + elif XML_NAME.match(entry_key): + xml += record_element(entry_key, entry_value) + xml += "" self.share_data.append(xml) @@ -380,6 +371,37 @@ def is_valid_xml_char(code): return any(low <= code <= high for low, high in VALID_XML_RANGES) +# The characters XML 1.0 allows in a name, without the colon, which names a +# namespace. +XML_NAME_START = ( + "A-Z_a-z\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u02ff\u0370-\u037d" + "\u037f-\u1fff\u200c-\u200d\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff" + "\uf900-\ufdcf\ufdf0-\ufffd\U00010000-\U000effff") +XML_NAME = re.compile( + "[" + XML_NAME_START + "][" + XML_NAME_START + + "\\-.0-9\u00b7\u0300-\u036f\u203f-\u2040]*\\Z") + + +def record_element(tag, value, name=None): + """! + One element of a usage sharing record. The value, and the Name attribute + when there is one, are made to fit the XML by ReplacedString, and the + element is marked escaped or truncated when either changed. + """ + replaced = ReplacedString(value) + escaped = replaced.escaped + xml = "<" + tag + if name is not None: + replaced_name = ReplacedString(name) + escaped = escaped or replaced_name.escaped + xml += ' Name="' + replaced_name.result + '"' + if escaped: + xml += ' escaped="true"' + if replaced.truncated: + xml += ' truncated="true"' + return xml + ">" + replaced.result + "" + + class ReplacedString: """! Makes an Evidence value fit in the usage sharing XML. The value is cut to From dbdd9f12cbc3c150f6e11f7c26eefbc8041bbabc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 10:13:18 +0000 Subject: [PATCH 09/11] Import re apart from the imports the back-off draft adds --- .../src/fiftyone_pipeline_engines_fiftyone/share_usage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 176f208a..e258ebbb 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -21,11 +21,11 @@ # ********************************************************************* import random +import re import json import gzip import platform import datetime -import re from concurrent.futures import ThreadPoolExecutor from importlib.metadata import version From af217355e5a2193003b36f9a604ce5f603954161 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:34:03 +0000 Subject: [PATCH 10/11] Test that the session id and client IP carry no attributes --- .../tests/test_share_usage_record.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py index d4c1b783..93992946 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py @@ -38,12 +38,21 @@ def test_session_id_is_written_so_the_record_parses(self): self.record("Mozilla/5.0", session_id=session_id)) session = device.find("SessionId") self.assertEqual('a Date: Tue, 10 Nov 2026 19:34:04 +0000 Subject: [PATCH 11/11] Write the session id and client IP with no escaped or truncated attribute, as the record gives them none --- .../share_usage.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index ae77235f..d0a1f913 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -300,11 +300,11 @@ def add_to_share_usage(self, data): # --- write variable data # The SessionID used to track a series of requests - xml += record_element("SessionId", data.session_id) + xml += plain_element("SessionId", data.session_id) # The sequence number of the request in a series of requests. xml += f"{data.sequence}" # The client IP of the request - xml += record_element("ClientIP", data.client_ip) + xml += plain_element("ClientIP", data.client_ip) # The UTC date and time this entry was written, to the second. date = datetime.datetime.now(datetime.timezone.utc).strftime( @@ -420,6 +420,15 @@ def is_valid_xml_char(code): "\\-.0-9\u00b7\u0300-\u036f\u203f-\u2040]*\\Z") +def plain_element(tag, value): + """! + An element of a usage sharing record that is not an Evidence entry, with + its value made to fit the XML by ReplacedString and no attributes, as + the record gives these elements none. + """ + return "<" + tag + ">" + ReplacedString(value).result + "" + + def record_element(tag, value, name=None): """! One element of a usage sharing record. The value, and the Name attribute