From d92184d7ad94ee99e04c0caed13b2875d23408a8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 6 Nov 2026 09:03:26 +0000 Subject: [PATCH 1/4] Test which evidence usage sharing keeps and sends Usage sharing should share what the specification's accepted evidence section allows unless the caller asks to share everything: headers other than Cookie, cookies, session values and query string parameters starting 51D_, the parameters the caller names, and any other evidence. The email address, the salt for hashed email identifiers, the query string and the path are never shared. Naming the session cookie keeps it in evidence but never sends it, and an ordinary cookie must not raise an error once any option is set. The evidence filter must also remove keys no element accepts without raising "dictionary changed size during iteration". These tests fail on rc, where every key is shared by default. --- .../fiftyone_pipeline_core/tests/test_core.py | 18 ++ .../test_share_usage_evidencekeyfilter.py | 168 ++++++++++++ .../tests/test_usage.py | 239 ++++++++++++++++++ 3 files changed, 425 insertions(+) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py diff --git a/pipeline/fiftyone_pipeline_core/tests/test_core.py b/pipeline/fiftyone_pipeline_core/tests/test_core.py index f26acde6..21407b33 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_core.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_core.py @@ -58,6 +58,24 @@ def testEvidenceKeyFilter(self): nullEvidence = testPipeline.flowdata.evidence.get("header.other-evidence") self.assertTrue(nullEvidence == None) + # Test that evidence no element accepts is removed without an error. + # The JavaScript and set header elements accept every key, so they are + # left out. + def testEvidenceKeyFilterRemovesKeys(self): + + pipeline = (PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}))\ + .add(ExampleFlowElement1())\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "test") + flowdata.evidence.add("header.accept-encoding", "gzip") + flowdata.evidence.add("header.host", "example.com") + flowdata.process() + self.assertEqual( + flowdata.evidence.get_all(), {"header.user-agent": "test"}) + # Test Getter methods def testGet(self): diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py new file mode 100644 index 00000000..16184d52 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -0,0 +1,168 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_engines_fiftyone.share_usage_evidencekeyfilter import \ + ShareUsageEvidenceKeyFilter + + +class ShareUsageEvidenceKeyFilterTests(unittest.TestCase): + + def test_default_rules(self): + """! + Check that by default headers other than Cookie are shared, cookies, + session values and query string parameters only when their names + start with 51D_, and any other evidence. + """ + + keys = ShareUsageEvidenceKeyFilter() + + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + self.assertFalse(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.51d_screenpixelswidth")) + self.assertTrue(keys.filter_evidence_key("cookie.51D_ScreenPixelsWidth")) + self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) + self.assertFalse(keys.filter_evidence_key("cookie.a.b")) + self.assertTrue(keys.filter_evidence_key("session.51d_value")) + self.assertFalse(keys.filter_evidence_key("session.org.example.attribute")) + self.assertTrue(keys.filter_evidence_key("query.51d_gethighentropyvalues")) + self.assertFalse(keys.filter_evidence_key("query.password")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertTrue(keys.filter_evidence_key("server.client-ip")) + self.assertTrue(keys.filter_evidence_key("nodot")) + + def test_field_is_everything_after_the_first_dot(self): + """! + Check that a name containing a dot is matched as a whole against + the parameters to share. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertFalse(keys.filter_evidence_key("query.valid")) + self.assertFalse(keys.filter_evidence_key("query.other.name")) + + def test_lists_ignore_case(self): + """! + Check that the headers not to share and the parameters to share + are matched without regard to case. + """ + + keys = ShareUsageEvidenceKeyFilter( + query_whitelist=["Campaign"], + header_blacklist=["X-Secret"]) + + self.assertFalse(keys.filter_evidence_key("header.x-secret")) + self.assertTrue(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + + def test_no_parameter_list_shares_every_parameter(self): + """! + Check that None in place of the parameters to share shares every + parameter, and an empty list only those starting 51D_. + """ + + every = ShareUsageEvidenceKeyFilter(query_whitelist=None) + prefixed = ShareUsageEvidenceKeyFilter(query_whitelist=[]) + + self.assertTrue(every.filter_evidence_key("query.campaign")) + self.assertFalse(prefixed.filter_evidence_key("query.campaign")) + self.assertTrue(prefixed.filter_evidence_key("query.51d_campaign")) + + def test_session_cookie(self): + """! + Check that the session cookie is included only when asked for, and + that no other ordinary cookie is. + """ + + kept = ShareUsageEvidenceKeyFilter(cookie="SessionId", include_session=True) + sent = ShareUsageEvidenceKeyFilter(cookie="SessionId") + + self.assertTrue(kept.filter_evidence_key("cookie.sessionid")) + self.assertFalse(sent.filter_evidence_key("cookie.sessionid")) + self.assertFalse(kept.filter_evidence_key("cookie._ga")) + self.assertFalse(sent.filter_evidence_key("cookie._ga")) + + def test_share_all(self): + """! + Check that sharing everything shares the evidence the other rules + leave out, apart from the evidence that is never shared. + """ + + keys = ShareUsageEvidenceKeyFilter(share_all=True) + + self.assertTrue(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.sessionid")) + self.assertTrue(keys.filter_evidence_key("query.password")) + self.assertTrue(keys.filter_evidence_key("query.id.usage")) + self.assertTrue(keys.filter_evidence_key("server.request-method")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertFalse(keys.filter_evidence_key("header.id.email")) + self.assertFalse(keys.filter_evidence_key("query.id.salt")) + self.assertFalse(keys.filter_evidence_key("header.id.salt")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) + self.assertFalse(keys.filter_evidence_key("server.request-path")) + + def test_never_shared_in_every_mode(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are never shared, + whether bare or after a prefix, and without regard to case. + """ + + for keys in ( + ShareUsageEvidenceKeyFilter(), + ShareUsageEvidenceKeyFilter(query_whitelist=None), + ShareUsageEvidenceKeyFilter(share_all=True)): + for key in ( + "id.email", "id.salt", + "query.id.email", "QUERY.ID.EMAIL", + "header.id.salt", "HEADER.ID.SALT", + "server.request-query", "SERVER.REQUEST-QUERY", + "server.request-path", "SERVER.REQUEST-PATH"): + self.assertFalse(keys.filter_evidence_key(key), key) + + def test_never_shared_keys_match_whole_parts(self): + """! + Check that a key that only ends with the same letters as a key + never shared is treated as an ordinary key. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertTrue( + ShareUsageEvidenceKeyFilter(share_all=True) + .filter_evidence_key("query.valid.email")) + + def test_naming_a_parameter_does_not_share_the_query_string(self): + """! + Check that sharing one parameter does not share the whole query + string, which carries every parameter. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["utm_source"]) + + self.assertTrue(keys.filter_evidence_key("query.utm_source")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 8329645b..e2c93d0e 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -431,6 +431,245 @@ def test_includes_header(self): self.assertIn('&', app.received[0]) + def test_default_shares_only_accepted_evidence(self): + """! + Check that with no options the element shares headers and the + 51Degrees cookies and parameters, and leaves out other cookies, the + Cookie header, and other query string and form parameters. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.51d_screenpixelswidth", "1080") + flowdata.evidence.add("query.51d_gethighentropyvalues", "hev") + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("header.cookie", "sessionid=header value") + flowdata.evidence.add("query.password", "password value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('
some user agent
', app.received[0]) + self.assertIn('1080', app.received[0]) + self.assertIn('hev', app.received[0]) + self.assertNotIn("cookie value", app.received[0]) + self.assertNotIn("header value", app.received[0]) + self.assertNotIn("password value", app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_never_shared_evidence(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are not shared, though + the rules for their prefixes would share them. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("id.email", "email value") + flowdata.evidence.add("header.id.salt", "salt value") + flowdata.evidence.add("server.request-query", "query string value") + flowdata.evidence.add("server.request-path", "path value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("email value", app.received[0]) + self.assertNotIn("salt value", app.received[0]) + self.assertNotIn("query string value", app.received[0]) + self.assertNotIn("path value", app.received[0]) + + def test_share_all(self): + """! + Check that sharing all evidence shares what the other rules leave + out, but not the evidence that is never shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + share_all=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('cookie value', app.received[0]) + self.assertIn('campaign value', app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_parameter_names_with_dots(self): + """! + Check that a query string parameter whose name contains a dot is + matched by its whole name against the parameters to share. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=["valid.email"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.valid.email", "valid value") + flowdata.evidence.add("query.valid", "first part value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('valid value', app.received[0]) + self.assertNotIn("first part value", app.received[0]) + + def test_empty_query_whitelist(self): + """! + Check that an empty list of parameters to share shares only those + starting 51D_, and that None shares every parameter. + """ + + for query_whitelist, shared in (([], False), (None, True)): + app.received = [] + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=query_whitelist, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.51d_param", "fiftyone value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("fiftyone value", app.received[0]) + self.assertEqual("campaign value" in app.received[0], shared) + + def test_session_cookie(self): + """! + Check that naming the session cookie keeps it in the evidence, but + neither it nor any other ordinary cookie is shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid") + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.sessionid", "session value") + flowdata.evidence.add("cookie._ga", "analytics value") + flowdata.process() + + self.waitForUsageThreads() + self.assertTrue(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("session value", app.received[0]) + self.assertNotIn("analytics value", app.received[0]) + + def test_ordinary_cookie_with_header_blacklist(self): + """! + Check that an ordinary cookie is left out without an error when + some headers are not to be shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("header.x-secret", "secret value") + flowdata.evidence.add("cookie.csrftoken", "token value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("secret value", app.received[0]) + self.assertNotIn("token value", app.received[0]) + + def test_repeat_with_new_sequence_not_shared(self): + """! + Check that a request differing from one already shared only in + its session id and sequence number is not shared again. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session, sequence in (("session 1", "1"), ("session 2", "2")): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("query.session-id", session) + flowdata.evidence.add("query.sequence", sequence) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("session 1", app.received[0]) + + def test_evidence_without_a_prefix(self): + """! + Check that evidence whose key has no prefix is shared without an + error. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("nodot", "no prefix value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("no prefix value", app.received[0]) + # zzz is to make this test run last so no other tests depend on the receiver anymore # we depend on python unittests ordering tests alphabetically, which is not ideal, but works for now def test_zzz_timeout(self): From a97de64351dbb47bd9b7d281d1e042d9fff065b9 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 6 Nov 2026 09:09:12 +0000 Subject: [PATCH 2/4] Share only the evidence usage sharing accepts unless asked for all The usage sharing filter shared every piece of evidence whenever no cookie, parameter list or header list was given, which is how the builders add it, so every cookie, form field and query string parameter was sent. Sharing everything is now only done when share_all is True, which defaults to False, as the specification's share all evidence option does. An empty parameter list shares only parameters starting 51D_, and None shares them all. The filter now also - splits a key at its first dot, so a name containing a dot is matched whole, and a key with no dot is shared rather than raising IndexError, - never shares id.email, id.salt, request-query or request-path, in any mode, as .NET does, - does not share the Cookie header, which carries every cookie, unless all evidence is shared, - matches the header and parameter lists without regard to case, - sets include_session, whose absence raised AttributeError for any ordinary cookie once an option was set. ShareUsage builds three filters once, as .NET does. The pipeline keeps the named session cookie, the data sent never includes it, and the tracker leaves out the session id and sequence number, which made every request look new. The session id and sequence number stay in evidence because the usage data carries them separately. Evidence.filter() looped over the evidence while deleting from it, which raised "dictionary changed size during iteration" whenever it removed a key. It now loops over a copy of the keys. --- .../src/fiftyone_pipeline_core/evidence.py | 3 +- .../share_usage.py | 59 ++++--- .../share_usage_evidencekeyfilter.py | 161 +++++++++++------- 3 files changed, 141 insertions(+), 82 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index 627ca7d8..f7067647 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -98,7 +98,8 @@ def filter(self) -> None: if self.evidence_filtered: return - for key in self.evidence: + # Loop over a copy of the keys, as keys are removed on the way. + for key in list(self.evidence): if not any([ flow_element.filter_evidence_key(key) for flow_element diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 3173318e..7d4db4d1 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -31,7 +31,8 @@ import requests from fiftyone_pipeline_engines.engine import Engine -from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter +from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter, \ + ShareUsageTrackerEvidenceKeyFilter from .share_usage_tracker import ShareUsageTracker # The maximum length of a piece of evidence's value which can be @@ -54,7 +55,8 @@ def __init__( header_blacklist = [], share_percentage = 1, endpoint = "https://devices-v4.51degrees.com/new.ashx", - request_timeout=REQUEST_TIMEOUT): + request_timeout=REQUEST_TIMEOUT, + share_all = False): """! Constructor for ShareUsage element @@ -70,22 +72,27 @@ def __init__( reduce the overhead of sending HTTP messages. For example, the 51Degrees cloud service uses a value of 2500. @type cookie: string - @param cookie: If a cookie is being used to identify user - sessions, it can be specified here in order to reduce the - sharing of duplicated data. + @param cookie: If a cookie is being used to identify user + sessions, it can be specified here. Evidence already shared is + then shared again for a new session, and the cookie itself is + never shared. @type query_whitelist: list - @param query_whitelist: By default query string and HTTP form + @param query_whitelist: By default query string and HTTP form parameters are not shared unless prefixed with '51D_'. - If you need to share query string parameters, a list can be - specified here. - @type query_blacklist: list - @param query_blacklist: By default, all HTTP headers - (except a few, such as Cookies) are shared. - Individual headers can be excluded from sharing by adding them + If you need to share query string parameters, a list can be + specified here. None shares every parameter. + @type header_blacklist: list + @param header_blacklist: By default, all HTTP headers + except Cookie are shared. + Individual headers can be excluded from sharing by adding them to this list. @type share_percentage : float @param share_percentage: approximate proportion of requests to be shared. 1 = 100%, 0.5 = 50%, etc.. + @type share_all: bool + @param share_all: If True, all evidence is shared apart from the + email address, the salt used for hashed email identifiers, and the + request's query string and path. False by default. """ super(ShareUsage, self).__init__() @@ -102,6 +109,22 @@ def __init__( self.share_percentage = share_percentage self.endpoint = endpoint self.request_timeout = request_timeout + self.share_all = share_all + + # The session id and sequence number are kept in evidence, as the + # usage data carries them separately. + included = None if query_whitelist is None else \ + list(query_whitelist) + ["session-id", "sequence"] + + # The pipeline keeps the session cookie for the tracker, but the + # data sent never includes it. + self.evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, True, share_all) + self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, False, share_all) + self.tracker_evidence_key_filter = \ + ShareUsageTrackerEvidenceKeyFilter( + cookie, included, header_blacklist, True, share_all) # Add the share usage tracker which detects when # to send up sharing data @@ -188,11 +211,7 @@ def get_evidence_key_filter(self): """ - return ShareUsageEvidenceKeyFilter( - cookie = self.cookie, - query_whitelist = self.query_whitelist, - header_blacklist = self.header_blacklist - ) + return self.evidence_key_filter def share_send_usage(self): """! @@ -291,7 +310,7 @@ def process_internal(self, flow_data): """ if random.uniform(0, 1) <= self.share_percentage: - cachekey = self.get_evidence_key_filter().filter_evidence(flow_data.evidence.get_all()) + cachekey = self.tracker_evidence_key_filter.filter_evidence(flow_data.evidence.get_all()) cachekey = json.dumps(cachekey) share = self.tracker.track(cachekey) if share: @@ -327,7 +346,7 @@ def get_data_from_evidence(self, flow_data): f"The value '{value}' could not be parsed to an integer.") else: # Check if we can send this piece of evidence - if self.get_evidence_key_filter().filter_evidence_key(key): + if self.data_evidence_key_filter.filter_evidence_key(key): data.try_add(key, value) return data @@ -394,7 +413,7 @@ def try_add(self, key, value): category = "" field = key - first_separator = key.index(".") + first_separator = key.find(".") if first_separator > 0: category = key[0:first_separator] field = key[first_separator + 1:len(key)] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index a9d0d2ac..5f2292f3 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -22,41 +22,64 @@ from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +# Cookies, session values and query string parameters whose names start with +# this are the ones 51Degrees sets, and are shared. +FIFTYONE_PREFIX = "51d_" + +# The Cookie header carries every cookie, including the ones the cookie rule +# leaves out, so it is not shared unless all evidence is. +BLOCKED_HEADERS = ("cookie",) + +# Evidence that is never shared, whatever the settings, matched as the whole +# key or after a '.'. 'id.email' and 'id.salt' are a raw email address and the +# salt used to make hashed email identifiers. 'request-query' is the whole query +# string, which would undo the rules for its parameters. 'request-path' would +# make each page a visitor opens look like a new session to the tracker. +NEVER_SHARED_KEYS = ("id.email", "id.salt", "request-query", "request-path") + class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ - The ShareUsageEvidenceKeyFilter filters out all evidence - not needed by the 51Degrees ShareUsage service. - It allows for a specific whitelist of query strings, - a blacklist of headers and a specific cookie used for - session information + The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees + ShareUsage service is sent. Headers are shared unless they are blocked, + cookies and session values only when their names start with 51D_, and + query string and form parameters only when their names start with 51D_ or + are on the list to share. Any other evidence is shared. """ - def __init__(self, cookie = None, query_whitelist = [], header_blacklist = []): + def __init__( + self, + cookie = None, + query_whitelist = (), + header_blacklist = (), + include_session = False, + share_all = False): """! Constructor for ShareUsageEvidenceKeyFilter @type cookie: string - @param cookie: which cookie is used to track evidence + @param cookie: the name of the cookie that identifies the session @type query_whitelist: list - @param query_whitelist: list of query string whitelist evidence to keep - @type query_blacklist: list - @param query_blacklist: list of header evidence to exclude - @type - @param + @param query_whitelist: query string and form parameters to share as + well as those starting 51D_, or None to share every parameter + @type header_blacklist: list + @param header_blacklist: headers not to share + @type include_session: bool + @param include_session: whether the session cookie is included + @type share_all: bool + @param share_all: whether all evidence is shared, apart from the + evidence that is never shared """ - self.query_whitelist = query_whitelist + self.cookie = cookie.lower() if cookie else None - self.header_blacklist = header_blacklist + self.query_whitelist = None if query_whitelist is None else \ + [parameter.lower() for parameter in query_whitelist] - self.cookie = cookie + self.header_blacklist = \ + [header.lower() for header in header_blacklist or ()] - if cookie is None and \ - (query_whitelist is None or len(query_whitelist) == 0) and \ - (header_blacklist is None or len(header_blacklist) == 0): - self.share_all = True - else: - self.share_all = False + self.include_session = include_session + + self.share_all = share_all - def filter_evidence_key(self, key): """! @@ -69,42 +92,58 @@ def filter_evidence_key(self, key): @return: Is this key in the filter's keys list? """ - # get prefix and key of evidence - key_parts = key.lower().split(".") - - prefix = key_parts[0] - suffix = key_parts[1] - - result = self.share_all - - if not self.share_all: - if prefix == "header": - # Add the header to the list if the header name does not - # appear in the list of blocked headers - result = suffix not in self.header_blacklist - elif prefix == "cookie": - # Only add cookies that start with the 51Degrees cookie - # prefix - result = suffix.startswith("51d_") or \ - (self.include_session and suffix == self.cookie) - elif prefix == "session": - # Only session values that start with the 51Degrees - # cookie prefix - result = suffix.startswith("51d_") - elif prefix == "query": - # If no query string parameter filter was specified - # then share all of them. - # Otherwise, only include query string parameters that - # start with 51d_ or that have been specified in - # the constructor. - result = self.query_whitelist is None or \ - len(self.query_whitelist) == 0 or \ - suffix.startswith("51d_") or \ - suffix in self.query_whitelist - else: - # Add anything that is not a cookie, header, session - # variable or query parameter - result = True - - return result - \ No newline at end of file + key = key.lower() + + if any(key == never_shared or key.endswith("." + never_shared) + for never_shared in NEVER_SHARED_KEYS): + return False + + if self.share_all: + return True + + # The prefix is everything before the first dot and the field + # everything after it, so a field can itself contain dots. + prefix, separator, field = key.partition(".") + + if not separator: + return True + elif prefix == "header": + # Add the header to the list if the header name does not + # appear in the list of blocked headers + return field not in self.header_blacklist and \ + field not in BLOCKED_HEADERS + elif prefix == "cookie": + # Only add cookies that start with the 51Degrees cookie + # prefix + return field.startswith(FIFTYONE_PREFIX) or \ + (self.include_session and field == self.cookie) + elif prefix == "session": + # Only session values that start with the 51Degrees + # cookie prefix + return field.startswith(FIFTYONE_PREFIX) + elif prefix == "query": + # If no query string parameter list was given then share all + # of them. Otherwise, only include query string parameters + # that start with 51d_ or that are in the list. + return self.query_whitelist is None or \ + field.startswith(FIFTYONE_PREFIX) or \ + field in self.query_whitelist + else: + # Add anything that is not a cookie, header, session + # variable or query parameter + return True + +class ShareUsageTrackerEvidenceKeyFilter(ShareUsageEvidenceKeyFilter): + """ + The evidence the ShareUsageTracker compares to decide whether a request + repeats one already shared. The session id and sequence number are left + out, because they make every request different. + """ + + def filter_evidence_key(self, key): + + if key.lower() in ("query.session-id", "query.sequence"): + return False + + return super(ShareUsageTrackerEvidenceKeyFilter, self)\ + .filter_evidence_key(key) From b885b9eb5990a3f9a3f58311d92be25c202bd144 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 00:52:04 +0000 Subject: [PATCH 3/4] Test that the session cookie is used only when session tracking is on .NET keeps two settings apart. SetTrackSession, false by default, decides whether the session cookie is used to tell sessions apart, and SetAspSessionCookieName only names the cookie, with no effect unless tracking is on. Here naming the cookie turned tracking on. The session cookie test now turns tracking on with track_session. Two new tests check that the cookie alone has no effect, and that with tracking on a repeat is shared again for a new session but not within the same session. --- .../tests/test_usage.py | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index e2c93d0e..4f1bea40 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -574,14 +574,16 @@ def test_empty_query_whitelist(self): def test_session_cookie(self): """! - Check that naming the session cookie keeps it in the evidence, but - neither it nor any other ordinary cookie is shared. + Check that with session tracking on the session cookie is kept in + the evidence, but neither it nor any other ordinary cookie is + shared. """ usage = ShareUsage( requested_package_size=1, endpoint=testEndpoint, - cookie="sessionid") + cookie="sessionid", + track_session=True) pipeline = (PipelineBuilder())\ .add(usage)\ @@ -599,6 +601,61 @@ def test_session_cookie(self): self.assertNotIn("session value", app.received[0]) self.assertNotIn("analytics value", app.received[0]) + def test_session_cookie_without_session_tracking(self): + """! + Check that session tracking is off by default, and that naming the + session cookie then has no effect. The cookie is not kept, and + evidence already shared is not shared again for a new session. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid") + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertNotIn("session value", app.received[0]) + self.assertFalse(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertFalse(usage.track_session) + + def test_session_tracking(self): + """! + Check that with session tracking on, evidence already shared is + shared again for a new session but not for a repeat in the same + session, and that the session cookie is not shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid", + track_session=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 2) + received = "\n".join(app.received) + self.assertEqual(received.count("repeated user agent"), 2) + self.assertNotIn("session value", received) + def test_ordinary_cookie_with_header_blacklist(self): """! Check that an ordinary cookie is left out without an error when From c7385d43a49967f9551e9222f2d999f392ff3bb5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 00:57:58 +0000 Subject: [PATCH 4/4] Track sessions only when asked, as .NET does ShareUsage takes track_session, False by default, as .NET's SetTrackSession is. The pipeline's filter and the tracker's filter include the session cookie named by cookie only when it is True, so the cookie has no effect otherwise, as .NET's SetAspSessionCookieName has none. The data sent still leaves the session cookie out unless share_all is True. The usage sharing example's configuration, which lists every option, now lists track_session and share_all with their defaults. --- .../examples/usagesharing/51d.json | 4 ++- .../share_usage.py | 27 ++++++++++++------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json index 90cfbada..7196a8cb 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json @@ -9,8 +9,10 @@ "interval": 1200, "requested_package_size": 50, "cookie": "", + "track_session": false, "query_whitelist": [], - "header_blacklist": [] + "header_blacklist": [], + "share_all": false } } ] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 7d4db4d1..210dfb0a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -56,7 +56,8 @@ def __init__( share_percentage = 1, endpoint = "https://devices-v4.51degrees.com/new.ashx", request_timeout=REQUEST_TIMEOUT, - share_all = False): + share_all = False, + track_session = False): """! Constructor for ShareUsage element @@ -72,10 +73,15 @@ def __init__( reduce the overhead of sending HTTP messages. For example, the 51Degrees cloud service uses a value of 2500. @type cookie: string - @param cookie: If a cookie is being used to identify user - sessions, it can be specified here. Evidence already shared is - then shared again for a new session, and the cookie itself is - never shared. + @param cookie: The name of the cookie that holds the session id. + It has no effect unless track_session is True. The cookie itself + is not shared unless share_all is True. + @type track_session: bool + @param track_session: If True, the cookie named by cookie is used + to tell user sessions apart, so evidence already shared is shared + again when it comes from a new session. False by default. It + needs cookie to be set, and has no effect when share_all is True, + because every cookie is then shared and used to find repeats. @type query_whitelist: list @param query_whitelist: By default query string and HTTP form parameters are not shared unless prefixed with '51D_'. @@ -110,21 +116,24 @@ def __init__( self.endpoint = endpoint self.request_timeout = request_timeout self.share_all = share_all + self.track_session = track_session # The session id and sequence number are kept in evidence, as the # usage data carries them separately. included = None if query_whitelist is None else \ list(query_whitelist) + ["session-id", "sequence"] - # The pipeline keeps the session cookie for the tracker, but the - # data sent never includes it. + # When sessions are tracked, the pipeline keeps the session cookie + # for the tracker. The data sent leaves it out unless all evidence + # is shared. self.evidence_key_filter = ShareUsageEvidenceKeyFilter( - cookie, included, header_blacklist, True, share_all) + cookie, included, header_blacklist, track_session, share_all) self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter( cookie, included, header_blacklist, False, share_all) self.tracker_evidence_key_filter = \ ShareUsageTrackerEvidenceKeyFilter( - cookie, included, header_blacklist, True, share_all) + cookie, included, header_blacklist, track_session, + share_all) # Add the share usage tracker which detects when # to send up sharing data