diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index 627ca7d8..f7067647 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -98,7 +98,8 @@ def filter(self) -> None: if self.evidence_filtered: return - for key in self.evidence: + # Loop over a copy of the keys, as keys are removed on the way. + for key in list(self.evidence): if not any([ flow_element.filter_evidence_key(key) for flow_element diff --git a/pipeline/fiftyone_pipeline_core/tests/test_core.py b/pipeline/fiftyone_pipeline_core/tests/test_core.py index f26acde6..21407b33 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_core.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_core.py @@ -58,6 +58,24 @@ def testEvidenceKeyFilter(self): nullEvidence = testPipeline.flowdata.evidence.get("header.other-evidence") self.assertTrue(nullEvidence == None) + # Test that evidence no element accepts is removed without an error. + # The JavaScript and set header elements accept every key, so they are + # left out. + def testEvidenceKeyFilterRemovesKeys(self): + + pipeline = (PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}))\ + .add(ExampleFlowElement1())\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "test") + flowdata.evidence.add("header.accept-encoding", "gzip") + flowdata.evidence.add("header.host", "example.com") + flowdata.process() + self.assertEqual( + flowdata.evidence.get_all(), {"header.user-agent": "test"}) + # Test Getter methods def testGet(self): diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json index 90cfbada..7196a8cb 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json @@ -9,8 +9,10 @@ "interval": 1200, "requested_package_size": 50, "cookie": "", + "track_session": false, "query_whitelist": [], - "header_blacklist": [] + "header_blacklist": [], + "share_all": false } } ] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 3173318e..210dfb0a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -31,7 +31,8 @@ import requests from fiftyone_pipeline_engines.engine import Engine -from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter +from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter, \ + ShareUsageTrackerEvidenceKeyFilter from .share_usage_tracker import ShareUsageTracker # The maximum length of a piece of evidence's value which can be @@ -54,7 +55,9 @@ def __init__( header_blacklist = [], share_percentage = 1, endpoint = "https://devices-v4.51degrees.com/new.ashx", - request_timeout=REQUEST_TIMEOUT): + request_timeout=REQUEST_TIMEOUT, + share_all = False, + track_session = False): """! Constructor for ShareUsage element @@ -70,22 +73,32 @@ def __init__( reduce the overhead of sending HTTP messages. For example, the 51Degrees cloud service uses a value of 2500. @type cookie: string - @param cookie: If a cookie is being used to identify user - sessions, it can be specified here in order to reduce the - sharing of duplicated data. + @param cookie: The name of the cookie that holds the session id. + It has no effect unless track_session is True. The cookie itself + is not shared unless share_all is True. + @type track_session: bool + @param track_session: If True, the cookie named by cookie is used + to tell user sessions apart, so evidence already shared is shared + again when it comes from a new session. False by default. It + needs cookie to be set, and has no effect when share_all is True, + because every cookie is then shared and used to find repeats. @type query_whitelist: list - @param query_whitelist: By default query string and HTTP form + @param query_whitelist: By default query string and HTTP form parameters are not shared unless prefixed with '51D_'. - If you need to share query string parameters, a list can be - specified here. - @type query_blacklist: list - @param query_blacklist: By default, all HTTP headers - (except a few, such as Cookies) are shared. - Individual headers can be excluded from sharing by adding them + If you need to share query string parameters, a list can be + specified here. None shares every parameter. + @type header_blacklist: list + @param header_blacklist: By default, all HTTP headers + except Cookie are shared. + Individual headers can be excluded from sharing by adding them to this list. @type share_percentage : float @param share_percentage: approximate proportion of requests to be shared. 1 = 100%, 0.5 = 50%, etc.. + @type share_all: bool + @param share_all: If True, all evidence is shared apart from the + email address, the salt used for hashed email identifiers, and the + request's query string and path. False by default. """ super(ShareUsage, self).__init__() @@ -102,6 +115,25 @@ def __init__( self.share_percentage = share_percentage self.endpoint = endpoint self.request_timeout = request_timeout + self.share_all = share_all + self.track_session = track_session + + # The session id and sequence number are kept in evidence, as the + # usage data carries them separately. + included = None if query_whitelist is None else \ + list(query_whitelist) + ["session-id", "sequence"] + + # When sessions are tracked, the pipeline keeps the session cookie + # for the tracker. The data sent leaves it out unless all evidence + # is shared. + self.evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, track_session, share_all) + self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, False, share_all) + self.tracker_evidence_key_filter = \ + ShareUsageTrackerEvidenceKeyFilter( + cookie, included, header_blacklist, track_session, + share_all) # Add the share usage tracker which detects when # to send up sharing data @@ -188,11 +220,7 @@ def get_evidence_key_filter(self): """ - return ShareUsageEvidenceKeyFilter( - cookie = self.cookie, - query_whitelist = self.query_whitelist, - header_blacklist = self.header_blacklist - ) + return self.evidence_key_filter def share_send_usage(self): """! @@ -291,7 +319,7 @@ def process_internal(self, flow_data): """ if random.uniform(0, 1) <= self.share_percentage: - cachekey = self.get_evidence_key_filter().filter_evidence(flow_data.evidence.get_all()) + cachekey = self.tracker_evidence_key_filter.filter_evidence(flow_data.evidence.get_all()) cachekey = json.dumps(cachekey) share = self.tracker.track(cachekey) if share: @@ -327,7 +355,7 @@ def get_data_from_evidence(self, flow_data): f"The value '{value}' could not be parsed to an integer.") else: # Check if we can send this piece of evidence - if self.get_evidence_key_filter().filter_evidence_key(key): + if self.data_evidence_key_filter.filter_evidence_key(key): data.try_add(key, value) return data @@ -394,7 +422,7 @@ def try_add(self, key, value): category = "" field = key - first_separator = key.index(".") + first_separator = key.find(".") if first_separator > 0: category = key[0:first_separator] field = key[first_separator + 1:len(key)] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index a9d0d2ac..5f2292f3 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -22,41 +22,64 @@ from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +# Cookies, session values and query string parameters whose names start with +# this are the ones 51Degrees sets, and are shared. +FIFTYONE_PREFIX = "51d_" + +# The Cookie header carries every cookie, including the ones the cookie rule +# leaves out, so it is not shared unless all evidence is. +BLOCKED_HEADERS = ("cookie",) + +# Evidence that is never shared, whatever the settings, matched as the whole +# key or after a '.'. 'id.email' and 'id.salt' are a raw email address and the +# salt used to make hashed email identifiers. 'request-query' is the whole query +# string, which would undo the rules for its parameters. 'request-path' would +# make each page a visitor opens look like a new session to the tracker. +NEVER_SHARED_KEYS = ("id.email", "id.salt", "request-query", "request-path") + class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ - The ShareUsageEvidenceKeyFilter filters out all evidence - not needed by the 51Degrees ShareUsage service. - It allows for a specific whitelist of query strings, - a blacklist of headers and a specific cookie used for - session information + The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees + ShareUsage service is sent. Headers are shared unless they are blocked, + cookies and session values only when their names start with 51D_, and + query string and form parameters only when their names start with 51D_ or + are on the list to share. Any other evidence is shared. """ - def __init__(self, cookie = None, query_whitelist = [], header_blacklist = []): + def __init__( + self, + cookie = None, + query_whitelist = (), + header_blacklist = (), + include_session = False, + share_all = False): """! Constructor for ShareUsageEvidenceKeyFilter @type cookie: string - @param cookie: which cookie is used to track evidence + @param cookie: the name of the cookie that identifies the session @type query_whitelist: list - @param query_whitelist: list of query string whitelist evidence to keep - @type query_blacklist: list - @param query_blacklist: list of header evidence to exclude - @type - @param + @param query_whitelist: query string and form parameters to share as + well as those starting 51D_, or None to share every parameter + @type header_blacklist: list + @param header_blacklist: headers not to share + @type include_session: bool + @param include_session: whether the session cookie is included + @type share_all: bool + @param share_all: whether all evidence is shared, apart from the + evidence that is never shared """ - self.query_whitelist = query_whitelist + self.cookie = cookie.lower() if cookie else None - self.header_blacklist = header_blacklist + self.query_whitelist = None if query_whitelist is None else \ + [parameter.lower() for parameter in query_whitelist] - self.cookie = cookie + self.header_blacklist = \ + [header.lower() for header in header_blacklist or ()] - if cookie is None and \ - (query_whitelist is None or len(query_whitelist) == 0) and \ - (header_blacklist is None or len(header_blacklist) == 0): - self.share_all = True - else: - self.share_all = False + self.include_session = include_session + + self.share_all = share_all - def filter_evidence_key(self, key): """! @@ -69,42 +92,58 @@ def filter_evidence_key(self, key): @return: Is this key in the filter's keys list? """ - # get prefix and key of evidence - key_parts = key.lower().split(".") - - prefix = key_parts[0] - suffix = key_parts[1] - - result = self.share_all - - if not self.share_all: - if prefix == "header": - # Add the header to the list if the header name does not - # appear in the list of blocked headers - result = suffix not in self.header_blacklist - elif prefix == "cookie": - # Only add cookies that start with the 51Degrees cookie - # prefix - result = suffix.startswith("51d_") or \ - (self.include_session and suffix == self.cookie) - elif prefix == "session": - # Only session values that start with the 51Degrees - # cookie prefix - result = suffix.startswith("51d_") - elif prefix == "query": - # If no query string parameter filter was specified - # then share all of them. - # Otherwise, only include query string parameters that - # start with 51d_ or that have been specified in - # the constructor. - result = self.query_whitelist is None or \ - len(self.query_whitelist) == 0 or \ - suffix.startswith("51d_") or \ - suffix in self.query_whitelist - else: - # Add anything that is not a cookie, header, session - # variable or query parameter - result = True - - return result - \ No newline at end of file + key = key.lower() + + if any(key == never_shared or key.endswith("." + never_shared) + for never_shared in NEVER_SHARED_KEYS): + return False + + if self.share_all: + return True + + # The prefix is everything before the first dot and the field + # everything after it, so a field can itself contain dots. + prefix, separator, field = key.partition(".") + + if not separator: + return True + elif prefix == "header": + # Add the header to the list if the header name does not + # appear in the list of blocked headers + return field not in self.header_blacklist and \ + field not in BLOCKED_HEADERS + elif prefix == "cookie": + # Only add cookies that start with the 51Degrees cookie + # prefix + return field.startswith(FIFTYONE_PREFIX) or \ + (self.include_session and field == self.cookie) + elif prefix == "session": + # Only session values that start with the 51Degrees + # cookie prefix + return field.startswith(FIFTYONE_PREFIX) + elif prefix == "query": + # If no query string parameter list was given then share all + # of them. Otherwise, only include query string parameters + # that start with 51d_ or that are in the list. + return self.query_whitelist is None or \ + field.startswith(FIFTYONE_PREFIX) or \ + field in self.query_whitelist + else: + # Add anything that is not a cookie, header, session + # variable or query parameter + return True + +class ShareUsageTrackerEvidenceKeyFilter(ShareUsageEvidenceKeyFilter): + """ + The evidence the ShareUsageTracker compares to decide whether a request + repeats one already shared. The session id and sequence number are left + out, because they make every request different. + """ + + def filter_evidence_key(self, key): + + if key.lower() in ("query.session-id", "query.sequence"): + return False + + return super(ShareUsageTrackerEvidenceKeyFilter, self)\ + .filter_evidence_key(key) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py new file mode 100644 index 00000000..16184d52 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -0,0 +1,168 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_engines_fiftyone.share_usage_evidencekeyfilter import \ + ShareUsageEvidenceKeyFilter + + +class ShareUsageEvidenceKeyFilterTests(unittest.TestCase): + + def test_default_rules(self): + """! + Check that by default headers other than Cookie are shared, cookies, + session values and query string parameters only when their names + start with 51D_, and any other evidence. + """ + + keys = ShareUsageEvidenceKeyFilter() + + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + self.assertFalse(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.51d_screenpixelswidth")) + self.assertTrue(keys.filter_evidence_key("cookie.51D_ScreenPixelsWidth")) + self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) + self.assertFalse(keys.filter_evidence_key("cookie.a.b")) + self.assertTrue(keys.filter_evidence_key("session.51d_value")) + self.assertFalse(keys.filter_evidence_key("session.org.example.attribute")) + self.assertTrue(keys.filter_evidence_key("query.51d_gethighentropyvalues")) + self.assertFalse(keys.filter_evidence_key("query.password")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertTrue(keys.filter_evidence_key("server.client-ip")) + self.assertTrue(keys.filter_evidence_key("nodot")) + + def test_field_is_everything_after_the_first_dot(self): + """! + Check that a name containing a dot is matched as a whole against + the parameters to share. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertFalse(keys.filter_evidence_key("query.valid")) + self.assertFalse(keys.filter_evidence_key("query.other.name")) + + def test_lists_ignore_case(self): + """! + Check that the headers not to share and the parameters to share + are matched without regard to case. + """ + + keys = ShareUsageEvidenceKeyFilter( + query_whitelist=["Campaign"], + header_blacklist=["X-Secret"]) + + self.assertFalse(keys.filter_evidence_key("header.x-secret")) + self.assertTrue(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + + def test_no_parameter_list_shares_every_parameter(self): + """! + Check that None in place of the parameters to share shares every + parameter, and an empty list only those starting 51D_. + """ + + every = ShareUsageEvidenceKeyFilter(query_whitelist=None) + prefixed = ShareUsageEvidenceKeyFilter(query_whitelist=[]) + + self.assertTrue(every.filter_evidence_key("query.campaign")) + self.assertFalse(prefixed.filter_evidence_key("query.campaign")) + self.assertTrue(prefixed.filter_evidence_key("query.51d_campaign")) + + def test_session_cookie(self): + """! + Check that the session cookie is included only when asked for, and + that no other ordinary cookie is. + """ + + kept = ShareUsageEvidenceKeyFilter(cookie="SessionId", include_session=True) + sent = ShareUsageEvidenceKeyFilter(cookie="SessionId") + + self.assertTrue(kept.filter_evidence_key("cookie.sessionid")) + self.assertFalse(sent.filter_evidence_key("cookie.sessionid")) + self.assertFalse(kept.filter_evidence_key("cookie._ga")) + self.assertFalse(sent.filter_evidence_key("cookie._ga")) + + def test_share_all(self): + """! + Check that sharing everything shares the evidence the other rules + leave out, apart from the evidence that is never shared. + """ + + keys = ShareUsageEvidenceKeyFilter(share_all=True) + + self.assertTrue(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.sessionid")) + self.assertTrue(keys.filter_evidence_key("query.password")) + self.assertTrue(keys.filter_evidence_key("query.id.usage")) + self.assertTrue(keys.filter_evidence_key("server.request-method")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertFalse(keys.filter_evidence_key("header.id.email")) + self.assertFalse(keys.filter_evidence_key("query.id.salt")) + self.assertFalse(keys.filter_evidence_key("header.id.salt")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) + self.assertFalse(keys.filter_evidence_key("server.request-path")) + + def test_never_shared_in_every_mode(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are never shared, + whether bare or after a prefix, and without regard to case. + """ + + for keys in ( + ShareUsageEvidenceKeyFilter(), + ShareUsageEvidenceKeyFilter(query_whitelist=None), + ShareUsageEvidenceKeyFilter(share_all=True)): + for key in ( + "id.email", "id.salt", + "query.id.email", "QUERY.ID.EMAIL", + "header.id.salt", "HEADER.ID.SALT", + "server.request-query", "SERVER.REQUEST-QUERY", + "server.request-path", "SERVER.REQUEST-PATH"): + self.assertFalse(keys.filter_evidence_key(key), key) + + def test_never_shared_keys_match_whole_parts(self): + """! + Check that a key that only ends with the same letters as a key + never shared is treated as an ordinary key. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertTrue( + ShareUsageEvidenceKeyFilter(share_all=True) + .filter_evidence_key("query.valid.email")) + + def test_naming_a_parameter_does_not_share_the_query_string(self): + """! + Check that sharing one parameter does not share the whole query + string, which carries every parameter. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["utm_source"]) + + self.assertTrue(keys.filter_evidence_key("query.utm_source")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 8329645b..4f1bea40 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -431,6 +431,302 @@ def test_includes_header(self): self.assertIn('&', app.received[0]) + def test_default_shares_only_accepted_evidence(self): + """! + Check that with no options the element shares headers and the + 51Degrees cookies and parameters, and leaves out other cookies, the + Cookie header, and other query string and form parameters. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.51d_screenpixelswidth", "1080") + flowdata.evidence.add("query.51d_gethighentropyvalues", "hev") + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("header.cookie", "sessionid=header value") + flowdata.evidence.add("query.password", "password value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('
some user agent
', app.received[0]) + self.assertIn('1080', app.received[0]) + self.assertIn('hev', app.received[0]) + self.assertNotIn("cookie value", app.received[0]) + self.assertNotIn("header value", app.received[0]) + self.assertNotIn("password value", app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_never_shared_evidence(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are not shared, though + the rules for their prefixes would share them. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("id.email", "email value") + flowdata.evidence.add("header.id.salt", "salt value") + flowdata.evidence.add("server.request-query", "query string value") + flowdata.evidence.add("server.request-path", "path value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("email value", app.received[0]) + self.assertNotIn("salt value", app.received[0]) + self.assertNotIn("query string value", app.received[0]) + self.assertNotIn("path value", app.received[0]) + + def test_share_all(self): + """! + Check that sharing all evidence shares what the other rules leave + out, but not the evidence that is never shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + share_all=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('cookie value', app.received[0]) + self.assertIn('campaign value', app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_parameter_names_with_dots(self): + """! + Check that a query string parameter whose name contains a dot is + matched by its whole name against the parameters to share. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=["valid.email"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.valid.email", "valid value") + flowdata.evidence.add("query.valid", "first part value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('valid value', app.received[0]) + self.assertNotIn("first part value", app.received[0]) + + def test_empty_query_whitelist(self): + """! + Check that an empty list of parameters to share shares only those + starting 51D_, and that None shares every parameter. + """ + + for query_whitelist, shared in (([], False), (None, True)): + app.received = [] + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=query_whitelist, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.51d_param", "fiftyone value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("fiftyone value", app.received[0]) + self.assertEqual("campaign value" in app.received[0], shared) + + def test_session_cookie(self): + """! + Check that with session tracking on the session cookie is kept in + the evidence, but neither it nor any other ordinary cookie is + shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid", + track_session=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.sessionid", "session value") + flowdata.evidence.add("cookie._ga", "analytics value") + flowdata.process() + + self.waitForUsageThreads() + self.assertTrue(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("session value", app.received[0]) + self.assertNotIn("analytics value", app.received[0]) + + def test_session_cookie_without_session_tracking(self): + """! + Check that session tracking is off by default, and that naming the + session cookie then has no effect. The cookie is not kept, and + evidence already shared is not shared again for a new session. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid") + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertNotIn("session value", app.received[0]) + self.assertFalse(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertFalse(usage.track_session) + + def test_session_tracking(self): + """! + Check that with session tracking on, evidence already shared is + shared again for a new session but not for a repeat in the same + session, and that the session cookie is not shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid", + track_session=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 2) + received = "\n".join(app.received) + self.assertEqual(received.count("repeated user agent"), 2) + self.assertNotIn("session value", received) + + def test_ordinary_cookie_with_header_blacklist(self): + """! + Check that an ordinary cookie is left out without an error when + some headers are not to be shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("header.x-secret", "secret value") + flowdata.evidence.add("cookie.csrftoken", "token value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("secret value", app.received[0]) + self.assertNotIn("token value", app.received[0]) + + def test_repeat_with_new_sequence_not_shared(self): + """! + Check that a request differing from one already shared only in + its session id and sequence number is not shared again. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session, sequence in (("session 1", "1"), ("session 2", "2")): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("query.session-id", session) + flowdata.evidence.add("query.sequence", sequence) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("session 1", app.received[0]) + + def test_evidence_without_a_prefix(self): + """! + Check that evidence whose key has no prefix is shared without an + error. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("nodot", "no prefix value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("no prefix value", app.received[0]) + # zzz is to make this test run last so no other tests depend on the receiver anymore # we depend on python unittests ordering tests alphabetically, which is not ideal, but works for now def test_zzz_timeout(self):