diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 3df2733df..996694614 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -36,12 +36,18 @@ /** * This filter is used by the {@link ShareUsageElement}. * It will include anything that is: - * 1) An HTTP header that is not blocked by the constructor parameter. + * 1) An HTTP header that is not blocked by the constructor parameter, other + * than Cookie. * 2) A cookie that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX} or is * the session cookie (if configured in the constructor). - * 3) An query string parameters that have been configured to be shared - * using the constructor parameter. - * 4) Not a header, cookie or query string parameter. + * 3) A session value that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX}. + * 4) A query string parameter that starts with + * {@link Constants#FIFTYONE_COOKIE_PREFIX} or has been configured to be + * shared using the constructor parameter. + * 5) Not a header, cookie, session value or query string parameter. + * The prefix of a key is everything before its first '.', and the field + * everything after it. An email address, the salt for hashed email + * identifiers, the query string and the path are never included. * * As this filter is generally inclusive, it will often cause far more * evidence to be passed into a pipeline than the engine-specific @@ -75,6 +81,21 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { */ private final Set includedQueryStringParams = new HashSet<>(); + /** + * Evidence keys that are never shared, matched without regard to case as + * the whole key or after a '.'. 'id.email' and 'id.salt' are a raw email + * address and the salt used to make hashed email identifiers. + * 'request-query' is the whole query string, which would undo the rules + * for its parameters. 'request-path' would make each page a visitor opens + * look like a new session to the tracker. + */ + private static final String[] NEVER_SHARED_EVIDENCE_KEYS = { + "id.email", + "id.salt", + "request-query", + "request-path" + }; + /** * Constructor. * @param blockedHttpHeaders a list of the names of the HTTP headers that @@ -92,9 +113,13 @@ public EvidenceKeyFilterShareUsage( List includedQueryStringParams, boolean includeSession, String sessionCookieName) { + // The Cookie header carries every cookie, including the ones the + // cookie rule leaves out. this.blockedHttpHeaders.add("cookies"); + this.blockedHttpHeaders.add("cookie"); this.includeSession = includeSession; - this.sessionCookieName = sessionCookieName; + this.sessionCookieName = sessionCookieName == null ? + null : sessionCookieName.toLowerCase(); for (String header : blockedHttpHeaders) { String lowerHeader = header.toLowerCase(); if (!this.blockedHttpHeaders.contains(lowerHeader)) { @@ -111,30 +136,42 @@ public EvidenceKeyFilterShareUsage( @Override public boolean include(String key) { + String lowerKey = key.toLowerCase(); + for (String neverShared : NEVER_SHARED_EVIDENCE_KEYS) { + if (lowerKey.equals(neverShared) || + lowerKey.endsWith(EVIDENCE_SEPERATOR + neverShared)) { + return false; + } + } + boolean result; - String[] parts = key.toLowerCase().split("\\."); - if (parts.length == 2) { - switch (parts[0]) { + int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); + if (separator > 0) { + String prefix = lowerKey.substring(0, separator); + String field = lowerKey.substring(separator + 1); + switch (prefix) { case EVIDENCE_HTTPHEADER_PREFIX: // Add the header to the list if the header name does not // appear in the list of blocked headers. - result = blockedHttpHeaders.contains(parts[1]) == false; + result = blockedHttpHeaders.contains(field) == false; break; case EVIDENCE_COOKIE_PREFIX: // Only add cookies that start with the 51Degrees cookie // prefix. - result = parts[1].startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || - (includeSession && parts[1].equals(sessionCookieName)); + result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + (includeSession && field.equals(sessionCookieName)); break; case EVIDENCE_SESSION_PREFIX: // Only add session values that start with the 51Degrees cookie // prefix. - result = parts[1].startsWith(Constants.FIFTYONE_COOKIE_PREFIX); + result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX); break; case EVIDENCE_QUERY_PREFIX: - // Only include query string parameters that have been - // specified in the constructor. - result = includedQueryStringParams.contains(parts[1]); + // Only include query string parameters that start with the + // 51Degrees prefix or have been specified in the + // constructor. + result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + includedQueryStringParams.contains(field); break; default: // Add anything that is not a cookie or a header. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 58a46d731..ddb8809ba 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -29,6 +29,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.configuration.CacheConfiguration; import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; +import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsageTracker; import fiftyone.pipeline.engines.fiftyone.exceptions.HttpException; import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker; import fiftyone.pipeline.engines.trackers.Tracker; @@ -440,7 +441,7 @@ protected ShareUsageBase( this.tracker = new ShareUsageTracker( cacheConfiguration, intervalMillis, - new EvidenceKeyFilterShareUsage( + new EvidenceKeyFilterShareUsageTracker( blockedHttpHeaders, includedQueryStringParameters, trackSession, sessionCookieName)); } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index bad9c1bbd..e376660cf 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -83,7 +83,8 @@ public ShareUsageBuilderBase(ILoggerFactory loggerFactory, Logger logger) { } /** - * By default, query string parameters are not shared. + * By default, only query string parameters that start with 51D_ are + * shared. *

* This setter replaces the current list, if any *

@@ -95,7 +96,7 @@ public ShareUsageBuilderBase(ILoggerFactory loggerFactory, Logger logger) { * @return this builder */ - @DefaultValue("No sharing") + @DefaultValue("No sharing unless the parameter starts with 51D_") public ShareUsageBuilderBase setIncludedQueryStringParameters( List queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -104,7 +105,8 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, query string parameters are not shared. + * By default, only query string parameters that start with 51D_ are + * shared. *

* This setter replaces the current list, if any *

@@ -115,7 +117,7 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( * query string parameters to include * @return this builder */ - @DefaultValue("No sharing") + @DefaultValue("No sharing unless the parameter starts with 51D_") public ShareUsageBuilderBase setIncludedQueryStringParameters( String queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -125,7 +127,8 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, query string parameters are not shared. + * By default, only query string parameters that start with 51D_ are + * shared. *

* This setting adds the name of a parameter to share with 51Degrees * if needed. @@ -133,7 +136,7 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( * query string parameter to include * @return this builder */ - @DefaultValue("No sharing") + @DefaultValue("No sharing unless the parameter starts with 51D_") public ShareUsageBuilderBase setIncludedQueryStringParameter( String queryStringParameterName) { includedQueryStringParameters.add(queryStringParameterName.trim()); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java new file mode 100644 index 000000000..6e6e17147 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java @@ -0,0 +1,209 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests which evidence {@link EvidenceKeyFilterShareUsage} lets usage sharing + * keep and send. + */ +public class EvidenceKeyFilterShareUsageTests { + + private static EvidenceKeyFilterShareUsage filter( + List blockedHttpHeaders, + List includedQueryStringParams, + boolean includeSession) { + return new EvidenceKeyFilterShareUsage( + blockedHttpHeaders, + includedQueryStringParams, + includeSession, + DEFAULT_SESSION_COOKIE_NAME); + } + + private static EvidenceKeyFilterShareUsage defaults() { + return filter( + Collections.emptyList(), + Collections.emptyList(), + false); + } + + /** + * By default headers other than Cookie are shared, cookies, session + * values and query string parameters only when their names start with + * 51D_, and any other evidence. + */ + @Test + public void defaultRules() { + EvidenceKeyFilterShareUsage filter = defaults(); + + assertTrue(filter.include("header.user-agent")); + assertFalse(filter.include("header.cookie")); + assertTrue(filter.include("cookie.51d_screenpixelswidth")); + assertTrue(filter.include("cookie.51D_ScreenPixelsWidth")); + assertFalse(filter.include("cookie.sessionid")); + assertFalse(filter.include("cookie.a.b")); + assertTrue(filter.include("session.51d_value")); + assertFalse(filter.include("session.org.example.attribute")); + assertTrue(filter.include("query.51d_gethighentropyvalues")); + assertTrue(filter.include("query.51D_Test")); + assertFalse(filter.include("query.password")); + assertFalse(filter.include("query.id.email")); + assertTrue(filter.include("server.client-ip")); + assertTrue(filter.include("nodot")); + } + + /** + * A name containing a dot is matched as a whole against the parameters to + * share. + */ + @Test + public void fieldIsEverythingAfterTheFirstDot() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.emptyList(), + Collections.singletonList("valid.email"), + false); + + assertTrue(filter.include("query.valid.email")); + assertFalse(filter.include("query.valid")); + assertFalse(filter.include("query.other.name")); + } + + /** + * The headers not to share and the parameters to share are matched + * without regard to case. + */ + @Test + public void listsIgnoreCase() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.singletonList("X-Secret"), + Collections.singletonList("Campaign"), + false); + + assertFalse(filter.include("header.x-secret")); + assertTrue(filter.include("query.campaign")); + assertTrue(filter.include("header.user-agent")); + } + + /** + * The session cookie is included only when asked for, whatever the case of + * its name, and no other ordinary cookie is. + */ + @Test + public void sessionCookie() { + EvidenceKeyFilterShareUsage kept = filter( + Collections.emptyList(), + Collections.emptyList(), + true); + EvidenceKeyFilterShareUsage sent = defaults(); + + assertTrue(kept.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertTrue(kept.include("cookie.jsessionid")); + assertFalse(sent.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertFalse(kept.include("cookie._ga")); + assertFalse(sent.include("cookie._ga")); + } + + /** + * The email address, the salt for hashed email identifiers, the query + * string and the path are never shared, whether bare or after a prefix, + * and without regard to case, even when the rules for their prefixes + * would share them. + */ + @Test + public void neverShared() { + List filters = Arrays.asList( + defaults(), + filter( + Collections.emptyList(), + Arrays.asList("id.email", "id.salt"), + true)); + List keys = Arrays.asList( + "id.email", "id.salt", + "query.id.email", "QUERY.ID.EMAIL", + "header.id.email", "header.id.salt", "HEADER.ID.SALT", + "server.request-query", "SERVER.REQUEST-QUERY", + "server.request-path", "SERVER.REQUEST-PATH"); + for (EvidenceKeyFilterShareUsage filter : filters) { + for (String key : keys) { + assertFalse(filter.include(key), key); + } + } + } + + /** + * A key that only ends with the same letters as a key never shared is + * treated as an ordinary key. + */ + @Test + public void neverSharedKeysMatchWholeParts() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.emptyList(), + Collections.singletonList("valid.email"), + false); + + assertTrue(filter.include("query.valid.email")); + } + + /** + * Sharing one parameter does not share the whole query string, which + * carries every parameter. + */ + @Test + public void namingAParameterDoesNotShareTheQueryString() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.emptyList(), + Collections.singletonList("utm_source"), + false); + + assertTrue(filter.include("query.utm_source")); + assertFalse(filter.include("server.request-query")); + } + + /** + * The tracker leaves out the session id and sequence number, which differ + * on every request, and follows the same rules otherwise. + */ + @Test + public void trackerLeavesOutSessionIdAndSequence() { + EvidenceKeyFilterShareUsageTracker tracker = + new EvidenceKeyFilterShareUsageTracker( + Collections.emptyList(), + Arrays.asList("session-id", "sequence"), + false, + DEFAULT_SESSION_COOKIE_NAME); + + assertFalse(tracker.include("query.session-id")); + assertFalse(tracker.include("query.sequence")); + assertTrue(tracker.include("header.user-agent")); + assertFalse(tracker.include("query.id.email")); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index 176a2a156..260118fe6 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -1076,4 +1076,142 @@ public void CheckForContinueAndResumeWhenQueueFull() throws Exception { } } + + /** + * Default settings share headers other than Cookie, the 51Degrees cookies + * and query string parameters, and other evidence, and leave out ordinary + * cookies, other parameters, session values and the evidence that is + * never shared. + */ + @Test + public void ShareUsageElement_DefaultSharesOnlyAcceptedEvidence() throws Exception { + // Arrange + createShareUsage( + 1, + 1, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "some user agent"); + evidenceData.put("cookie.51d_screenpixelswidth", "screen width"); + evidenceData.put("query.51D_Test", "fiftyone parameter"); + evidenceData.put("cookie.sessionid", "sessionid cookie"); + evidenceData.put("header.cookie", "cookie header"); + evidenceData.put("query.password", "password parameter"); + evidenceData.put("query.id.email", "email parameter"); + evidenceData.put("header.id.email", "email header"); + evidenceData.put("id.salt", "salt value"); + evidenceData.put("server.request-query", "whole query string"); + evidenceData.put("server.request-path", "request path"); + evidenceData.put("session.org.example.attribute", "session attribute"); + + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + + // Act + shareUsageElement.process(data); + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + } + + // Assert + String output = new String( + connector.getBaos().toByteArray(), StandardCharsets.UTF_8); + for (String shared : Arrays.asList( + "some user agent", "screen width", "fiftyone parameter")) { + assertTrue(output.contains(shared), shared); + } + for (String withheld : Arrays.asList( + "sessionid cookie", "cookie header", "password parameter", + "email parameter", "email header", "salt value", + "whole query string", "request path", "session attribute")) { + assertFalse(output.contains(withheld), withheld); + } + } + + /** + * The default tracker does not share again a request that differs from + * one already shared only in its session id and sequence number. + */ + @Test + public void ShareUsageElement_RepeatWithNewSequence_NotShared() throws Exception { + // Arrange + tracker = null; + createShareUsage( + 1, + 3, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + + // Act + for (int sequence = 1; sequence <= 2; sequence++) { + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "repeated user agent"); + evidenceData.put(Constants.EVIDENCE_SESSIONID, "session " + sequence); + evidenceData.put(Constants.EVIDENCE_SEQUENCE, sequence); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + shareUsageElement.process(data); + } + } + shareUsageElement.close(); + + // Assert + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(connector.getBaos().toByteArray())); + NodeList devices = (NodeList) xpath.compile("//Device") + .evaluate(doc, XPathConstants.NODESET); + assertEquals(1, devices.getLength(), "1 device expected"); + } + + /** + * The session cookie the element names is kept in evidence for the + * tracker, whatever the case of its name, and never sent. With sessions + * tracked, the same evidence from a new session is shared again. + */ + @Test + public void ShareUsageElement_TrackSession_SessionCookie() throws Exception { + // Arrange + tracker = null; + createShareUsage( + 1, + 3, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + String sessionCookie = EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + + DEFAULT_SESSION_COOKIE_NAME; + + // Act + for (int session = 1; session <= 2; session++) { + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "repeated user agent"); + evidenceData.put(sessionCookie, "session cookie " + session); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + shareUsageElement.process(data); + } + } + shareUsageElement.close(); + + // Assert + assertTrue(shareUsageElement.getEvidenceKeyFilter().include(sessionCookie)); + byte[] output = connector.getBaos().toByteArray(); + assertFalse(new String(output, StandardCharsets.UTF_8).contains( + "session cookie"), "the session cookie is never sent"); + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(output)); + NodeList devices = (NodeList) xpath.compile("//Device") + .evaluate(doc, XPathConstants.NODESET); + assertEquals(2, devices.getLength(), "1 device expected for each session"); + } }