From e4881375e61e593832df8dcc20ceea1861d81699 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 20:33:30 +0000 Subject: [PATCH 1/8] Test that the cloud request carries every Evidence value but the reserved names The Cloud Request Engine must send every Evidence value apart from the reserved names resource, license, licence and values, under any prefix and in any letter case, whether or not the cloud service lists the key in its evidencekeys answer. The test that only accepted Evidence is sent is replaced by one that the request carries Evidence the service does not list, and still leaves out reserved names under prefixes it does not list. --- .../flowelements/CloudRequestEngineTests.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 9e9f3a00..d0a7eccf 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -631,20 +631,23 @@ public void process_PostsWholeEvidenceName() throws Exception { } /** - * Only the Evidence whose keys the cloud service accepts is sent, with - * the keys compared without regard to case. + * Every Evidence value is sent, including Evidence the cloud service + * does not list as accepted, apart from the reserved names under any + * prefix. * See the Processing section of the cloud request engine specification. */ @Test - public void process_SendsOnlyAcceptedEvidence() throws Exception { - acceptEvidence(Arrays.asList( - "query.User-Agent", "header.Accept-Language")); + public void process_SendsEveryEvidenceValueButTheReservedNames() + throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); Map evidence = new HashMap<>(); evidence.put("query.User-Agent", "iPhone"); evidence.put("header.accept-language", "en-GB"); evidence.put("header.X-Other", "other"); evidence.put("cookie.session", "abc"); evidence.put("server.client-ip", "1.2.3.4"); + evidence.put("other.VALUES", "ismobile"); + evidence.put("server.licence", "evidence-licence"); List items = postedItems(null, evidence); @@ -652,10 +655,13 @@ public void process_SendsOnlyAcceptedEvidence() throws Exception { new HashSet<>(Arrays.asList( "resource=resourcekey", "User-Agent=iPhone", - "accept-language=en-GB")), + "accept-language=en-GB", + "X-Other=other", + "session=abc", + "client-ip=1.2.3.4")), new HashSet<>(items), - "The request carried Evidence the cloud service does not " + - "accept: " + items); + "The request did not carry every Evidence value apart from " + + "the reserved names: " + items); } /** From 4e73db5bbd049d7ca6b18e85c687d21a4e4f5234 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 20:37:18 +0000 Subject: [PATCH 2/8] Send every Evidence value but the reserved names in the cloud request The Cloud Request Engine sent only the Evidence whose keys the cloud service lists in its evidencekeys answer. It now sends every Evidence value apart from the reserved names resource, license, licence and values, under any prefix and in any letter case, and apart from the web request, as .NET does. What the service uses changes over time, and knowing about the other Evidence helps it improve. The accepted Evidence still serves as the engine's Evidence key filter, for the Pipeline and the web Vary header. --- .../flowelements/CloudRequestEngineDefault.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 1ca5d11c..5db67932 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -497,16 +497,15 @@ Map getFormData(FlowData flowData) { } /** - * Whether Evidence is sent to the cloud service. Only keys the service - * accepts are sent, never one that strips to a name in - * {@link #ENGINE_FIELDS}, compared without regard to case, and never - * the web request. + * Whether Evidence is sent to the cloud service. Every key is sent, + * including keys the service does not list as accepted, apart from one + * that strips to a name in {@link #ENGINE_FIELDS}, compared without + * regard to case, and the web request. * @param key the Evidence key * @return true if the Evidence is sent */ private boolean isSent(String key) { - if (HTTP_REQUEST_KEY.equalsIgnoreCase(key) || - evidenceKeyFilter.include(key) == false) { + if (HTTP_REQUEST_KEY.equalsIgnoreCase(key)) { return false; } String[] parts = key.split( From 6cfbd89617cb406e2988725a15639c7b9deb03f6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 07:39:29 +0000 Subject: [PATCH 3/8] Test that Evidence with no prefix is sent, and that prefixes ignore case Keys only usually carry a prefix, as the Overview of the Evidence specification says, and .NET sends a key with no prefix under its whole key unless it is a reserved name. Java drops it. .NET also compares prefixes without regard to case, where Java treats Query.User-Agent as Evidence with some other prefix, so header Evidence of the same name takes precedence over it with a warning. --- .../flowelements/CloudRequestEngineTests.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index d0a7eccf..c5454534 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -664,6 +664,61 @@ public void process_SendsEveryEvidenceValueButTheReservedNames() "the reserved names: " + items); } + /** + * Evidence whose key has no prefix is sent under its whole key, unless + * the key is a reserved name in any letter case. Keys only usually + * carry a prefix, as the Overview of the Evidence specification says. + */ + @Test + public void process_SendsEvidenceWithNoPrefixUnderItsWholeKey() + throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("query.User-Agent", "iPhone"); + evidence.put("bare", "value"); + evidence.put("Values", "ismobile"); + evidence.put("license", "evidence-license"); + evidence.put("RESOURCE", "evidence-resource"); + evidence.put("licence", "evidence-licence"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "User-Agent=iPhone", + "bare=value")), + new HashSet<>(items), + "The request did not carry the Evidence with no prefix, or " + + "carried a reserved name: " + items); + } + + /** + * Prefixes are compared without regard to case, so query Evidence + * takes precedence over header Evidence of the same name, with no + * warning, whatever the case of its prefix. Keys are case-insensitive, + * as the Overview of the Evidence specification says. + */ + @Test + public void process_ComparesPrefixesWithoutRegardToCase() + throws Exception { + loggerFactory.loggers.clear(); + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("Query.User-Agent", "query-value"); + evidence.put("header.User-Agent", "header-value"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "User-Agent=query-value")), + new HashSet<>(items), + "The query Evidence did not take precedence: " + items); + loggerFactory.assertMaxWarnings(0); + } + /** * Evidence whose key strips to resource, license or values is never * sent, whatever its case and even where the cloud service lists the From e6404fc8cfbf0659d689aa8fe0d32819a2182a4a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 07:40:39 +0000 Subject: [PATCH 4/8] Send Evidence with no prefix under its whole key, and compare prefixes ignoring case A key with no separator now has the whole key as both its prefix and the name it is sent under, as .NET's AddQueryData takes it, so it is sent unless it is a reserved name. Prefixes are compared without regard to case when Evidence is grouped and when a conflict is reported, as .NET's KeyHasPrefix and AddQueryData compare them. --- .../CloudRequestEngineDefault.java | 58 ++++++++++++------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 5db67932..b9b63ed4 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -498,20 +498,42 @@ Map getFormData(FlowData flowData) { /** * Whether Evidence is sent to the cloud service. Every key is sent, - * including keys the service does not list as accepted, apart from one - * that strips to a name in {@link #ENGINE_FIELDS}, compared without - * regard to case, and the web request. + * including keys the service does not list as accepted and keys with no + * prefix, apart from the web request and a key whose + * {@link #keySuffix(String)} is in {@link #ENGINE_FIELDS}, compared + * without regard to case. * @param key the Evidence key * @return true if the Evidence is sent */ private boolean isSent(String key) { - if (HTTP_REQUEST_KEY.equalsIgnoreCase(key)) { - return false; - } - String[] parts = key.split( - Pattern.quote(Constants.EVIDENCE_SEPERATOR), 2); - return parts.length == 2 && - ENGINE_FIELDS.contains(parts[1].toLowerCase(Locale.ROOT)) == false; + return HTTP_REQUEST_KEY.equalsIgnoreCase(key) == false && + ENGINE_FIELDS.contains( + keySuffix(key).toLowerCase(Locale.ROOT)) == false; + } + + /** + * The prefix of an Evidence key, being the part before its first + * separator, or the whole key where it has none. + * @param key the Evidence key + * @return the prefix + */ + private static String keyPrefix(String key) { + int separator = key.indexOf(Constants.EVIDENCE_SEPERATOR); + return separator >= 0 ? key.substring(0, separator) : key; + } + + /** + * The name an Evidence key is sent under, being the part after its + * first separator, or the whole key where it has none. The name can + * hold a dot itself, as id.usage does in query.id.usage. + * @param key the Evidence key + * @return the name to send + */ + private static String keySuffix(String key) { + int separator = key.indexOf(Constants.EVIDENCE_SEPERATOR); + return separator >= 0 ? + key.substring(separator + Constants.EVIDENCE_SEPERATOR.length()) : + key; } /** @@ -530,13 +552,8 @@ private void addFormData( Collections.sort(evidenceKeys, Collections.reverseOrder()); for (String evidenceKey : evidenceKeys) { - // Split at the first separator only. Everything after the prefix - // is the name the cloud service expects, and that name can hold - // a dot itself, as id.usage does in query.id.usage. - String[] evidenceKeyParts = evidenceKey.split( - Pattern.quote(Constants.EVIDENCE_SEPERATOR), 2); - String prefix = evidenceKeyParts[0]; - String suffix = evidenceKeyParts[1]; + String prefix = keyPrefix(evidenceKey); + String suffix = keySuffix(evidenceKey); // Check and add the evidence to the query parameters. if (formData.containsKey(suffix) == false) { @@ -548,7 +565,7 @@ private void addFormData( // warning, if the evidence prefix is not query. Otherwise a // warning is not needed as query evidence is expected // to overwrite any existing evidence with the same suffix. - if (prefix.equals("query") == false) { + if (prefix.equalsIgnoreCase("query") == false) { Map conflicts = new HashMap<>(); for (String key : allEvidence.keySet()) { if (key.equals(evidenceKey) == false && key.contains(suffix)) { @@ -606,13 +623,14 @@ Map getSelectedEvidence(Map evidence, String typ } /** - * Check that the key of a KeyValuePair has the given prefix. + * Check that the key of a KeyValuePair has the given prefix, compared + * without regard to case. * @param itemKey key to check * @param prefix the prefix to check for * @return true if the key has the prefix */ private boolean hasKeyPrefix(String itemKey, String prefix) { - return itemKey.startsWith(prefix + "."); + return keyPrefix(itemKey).equalsIgnoreCase(prefix); } @Override From ec1c22a461e58ca7c615368b110ca9efccf34a05 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 08:41:49 +0000 Subject: [PATCH 5/8] Test where Evidence with no prefix ranks in the cloud request Evidence with no prefix ranks with Evidence of any prefix other than query, header and cookie, so header Evidence of the same name takes precedence over it, and a bare key that is itself a prefix, such as query, ranks with that prefix's Evidence, as in .NET. The Evidence specification's precedence order covers prefixes only, so the test records the choice. --- .../flowelements/CloudRequestEngineTests.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index c5454534..755b427d 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -719,6 +719,34 @@ public void process_ComparesPrefixesWithoutRegardToCase() loggerFactory.assertMaxWarnings(0); } + /** + * Evidence with no prefix ranks with Evidence of any prefix other than + * query, header and cookie, so it has the lowest precedence, unless the + * whole key is one of those prefixes, which puts it with that prefix's + * Evidence. The precedence order of the Evidence specification orders + * prefixes only, so this follows .NET. + */ + @Test + public void process_RanksEvidenceWithNoPrefix() throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("X-Thing", "bare"); + evidence.put("header.X-Thing", "header"); + evidence.put("query", "bare-query"); + evidence.put("header.query", "header-query"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "X-Thing=header", + "query=bare-query")), + new HashSet<>(items), + "Evidence with no prefix was not ranked as .NET ranks it: " + + items); + } + /** * Evidence whose key strips to resource, license or values is never * sent, whatever its case and even where the cloud service lists the From e12820c46623b41b64312cce3b4a332e7f87154c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 08:42:58 +0000 Subject: [PATCH 6/8] Say where Evidence with no prefix ranks, and what hasKeyPrefix compares The getFormData comment now gives the precedence order and where Evidence with no prefix ranks in it, and the hasKeyPrefix comment says a key with no separator is its own prefix. --- .../flowelements/CloudRequestEngineDefault.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index b9b63ed4..2e257108 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -470,7 +470,11 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { /** * The Evidence to send, keyed by the name the cloud service expects. * Only Evidence that {@link #isSent(String)} allows is included, and - * where two prefixes give the same name the precedence order decides. + * where two prefixes give the same name the precedence order decides, + * query first, then header, then cookie, then any other prefix. + * Evidence with no prefix ranks with the other prefixes, unless the + * whole key is query, header or cookie, as {@link #keyPrefix(String)} + * gives the whole key as its prefix. * @param flowData the FlowData * @return the form fields to send */ @@ -623,8 +627,9 @@ Map getSelectedEvidence(Map evidence, String typ } /** - * Check that the key of a KeyValuePair has the given prefix, compared - * without regard to case. + * Check that an Evidence key has the given prefix, being the part + * before its first separator, or the whole key where it has none, + * compared without regard to case. * @param itemKey key to check * @param prefix the prefix to check for * @return true if the key has the prefix From 232101051158b85cb09531e3fc76c1ca5f492d2d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 09:22:14 +0000 Subject: [PATCH 7/8] Test that a repeated cloud request still hits the cache --- .../flowelements/CloudRequestEngineTests.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 755b427d..e7378bf8 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -816,6 +816,48 @@ public void process_LeavesOutEvidenceNamedLicence( assertEquals(expected, items); } + /** + * A cache key is built from the Evidence the engine's own filter + * accepts, and that filter still holds the keys the cloud service + * lists, so a repeated request finds the entry the first one wrote and + * one request is posted. The request carries Evidence the service does + * not list all the same. + * See the Generation of keys section of the caching specification. + */ + @Test + public void process_RepeatedRequestHitsTheCache() throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(10) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (int i = 0; i < 2; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.addEvidence("header.sec-ch-ua-platform", "Windows"); + data.process(); + } + } + } + + assertFalse( + engine.getEvidenceKeyFilter().include("header.sec-ch-ua-platform"), + "The engine accepts a key the cloud service does not list, so " + + "the cache key takes every Evidence value."); + ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); + verify(httpClient, times(1)).postData(any(), any(), body.capture()); + List items = Arrays.asList( + new String(body.getValue(), StandardCharsets.UTF_8).split("&")); + assertTrue(items.contains("sec-ch-ua-platform=Windows"), + "The request did not carry the Evidence the cloud service does " + + "not list: " + items); + } + /** * Answer the evidencekeys request with the given keys. * @param keys the Evidence keys the cloud service accepts From 7e8fac26a326b4c474411568ea159b6692ccdb98 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 09:29:23 +0000 Subject: [PATCH 8/8] Make the cache test's second request differ in Evidence the service does not list Two identical requests share a cache entry whatever the key is built from, so the test passed without showing that the key is built from the advertised Evidence. The second request now differs only in header.sec-ch-ua-platform, which the cloud service does not list, and one request must still be posted, carrying the first request's value. The test fails if the cache key takes every Evidence value. --- .../flowelements/CloudRequestEngineTests.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index e7378bf8..994408d1 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -817,11 +817,11 @@ public void process_LeavesOutEvidenceNamedLicence( } /** - * A cache key is built from the Evidence the engine's own filter - * accepts, and that filter still holds the keys the cloud service - * lists, so a repeated request finds the entry the first one wrote and - * one request is posted. The request carries Evidence the service does - * not list all the same. + * A cache key is built from the Evidence the engine advertises, which is + * still the keys the cloud service lists, so a second request that + * differs only in Evidence the service does not list finds the entry + * the first one wrote, and one request is posted. That request carries + * the Evidence the service does not list all the same. * See the Generation of keys section of the caching specification. */ @Test @@ -836,10 +836,10 @@ public void process_RepeatedRequestHitsTheCache() throws Exception { try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine).build()) { - for (int i = 0; i < 2; i++) { + for (String platform : Arrays.asList("Windows", "macOS")) { try (FlowData data = pipeline.createFlowData()) { data.addEvidence("query.User-Agent", "iPhone"); - data.addEvidence("header.sec-ch-ua-platform", "Windows"); + data.addEvidence("header.sec-ch-ua-platform", platform); data.process(); } } @@ -847,8 +847,7 @@ public void process_RepeatedRequestHitsTheCache() throws Exception { assertFalse( engine.getEvidenceKeyFilter().include("header.sec-ch-ua-platform"), - "The engine accepts a key the cloud service does not list, so " + - "the cache key takes every Evidence value."); + "The engine advertises a key the cloud service does not list."); ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); verify(httpClient, times(1)).postData(any(), any(), body.capture()); List items = Arrays.asList(