diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java index a4843828..d6e7d22d 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java @@ -50,11 +50,17 @@ private ShareUsageEvidence() { * Every Evidence value, and every value of the web request when the * Evidence holds it, keyed without regard to case. A key in both keeps * the Evidence value. The web request itself is not one of the values. + * Without a web request the values found are the Evidence's own + * read-only map, as Evidence is already keyed without regard to case and + * holds no null value, so nothing is copied. * @param evidence the Evidence of the request - * @return the values found + * @return the values found, which the caller must not change */ public static Map find(Evidence evidence) { Map values = evidence.asKeyMap(); + if (values.containsKey(HTTP_REQUEST_KEY) == false) { + return values; + } Map found = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); Object request = null; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 9e417dcf..1fd5bf08 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -645,12 +645,16 @@ public void addPipeline(Pipeline pipeline) { @Override protected void processInternal(FlowData flowData){ boolean ignoreData = false; - Map found = - ShareUsageEvidence.find(flowData.getEvidence()); + // The values are found here only for the ignore filter. Otherwise + // they are found once the tracker allows the call to be shared, as + // most calls repeat one already shared. + Map found = null; logger.debug("processInternal()"); - if (ignoreDataEvidenceFilter != null) { + if (ignoreDataEvidenceFilter != null && + ignoreDataEvidenceFilter.isEmpty() == false) { + found = ShareUsageEvidence.find(flowData.getEvidence()); for (Map.Entry entry : ignoreDataEvidenceFilter) { String key = entry.getKey(); if (found.containsKey(key)) { @@ -789,18 +793,21 @@ private boolean isLocalHost(String address) throws UnknownHostException { /** * Process the supplied request data * @param data the {@link FlowData} instance that provides the evidence - * @param found the values {@link ShareUsageEvidence#find} found for it + * @param found the values {@link ShareUsageEvidence#find} found for it, + * or null if they have not been found yet */ private void processData(FlowData data, Map found) { logger.debug("processData()"); if (random.nextDouble() <= sharePercentage) { // Check if the tracker will allow sharing of this data if (tracker.track(data)) { + Map shared = found != null ? found : + ShareUsageEvidence.find(data.getEvidence()); // Extract the data we want from the evidence and add // it to the collection. try { if (evidenceCollection.offer( - getDataFromEvidence(found), + getDataFromEvidence(shared), addTimeout, TimeUnit.MILLISECONDS)) { queueFullReported = false; diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java new file mode 100644 index 00000000..fee9b726 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java @@ -0,0 +1,117 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +/** + * The usage sharing element finds the values it shares on every request, so + * with no web request finding them reads the Evidence rather than copying + * every value of it. + */ +public class ShareUsageEvidenceTests { + + private static Pipeline createPipeline() throws Exception { + return new PipelineBuilder(LoggerFactory.getILoggerFactory()) + .addFlowElement(new EmptyEngineBuilder( + LoggerFactory.getILoggerFactory()).build()) + .setAutoCloseElements(true) + .build(); + } + + /** + * With no web request the values found are the Evidence itself, so a + * value added afterwards is among them. A copy would not hold it. + */ + @Test + public void Find_NoWebRequest_ReadsTheEvidenceWithoutCopying() + throws Exception { + try (Pipeline pipeline = createPipeline(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "evidence-ua"); + Map found = + ShareUsageEvidence.find(data.getEvidence()); + data.addEvidence("header.x-later", "later"); + + assertEquals("later", found.get("header.x-later"), + "The Evidence was copied, so every request pays for a copy " + + "of every Evidence value."); + } + } + + /** + * With no web request the values found are still keyed without regard + * to case. + */ + @Test + public void Find_NoWebRequest_KeyedWithoutRegardToCase() + throws Exception { + try (Pipeline pipeline = createPipeline(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.User-Agent", "evidence-ua"); + Map found = + ShareUsageEvidence.find(data.getEvidence()); + + assertEquals("evidence-ua", found.get("HEADER.USER-AGENT")); + assertEquals(1, found.size()); + } + } + + /** + * With a web request its values are added under their own keys, the + * Evidence value is kept for a key in both, and the web request itself + * is not one of the values. + */ + @Test + public void Find_WebRequest_AddsItsValuesAndKeepsTheEvidenceValue() + throws Exception { + Map request = new HashMap<>(); + request.put("header.User-Agent", "request-ua"); + request.put("header.X-Test", "shared"); + try (Pipeline pipeline = createPipeline(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "evidence-ua"); + data.addEvidence(ShareUsageEvidence.HTTP_REQUEST_KEY, + Collections.unmodifiableMap(request)); + Map found = + ShareUsageEvidence.find(data.getEvidence()); + + assertEquals("evidence-ua", found.get("HEADER.USER-AGENT")); + assertEquals("shared", found.get("header.x-test")); + assertFalse(found.containsKey(ShareUsageEvidence.HTTP_REQUEST_KEY)); + assertEquals(2, found.size()); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java new file mode 100644 index 00000000..d3d919e7 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java @@ -0,0 +1,187 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.testhelpers.data.MockFlowData; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.util.AbstractMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * Usage sharing is called for every request, and most calls repeat one + * already shared, which the tracker does not share again. Such a call must + * not copy every value of the Evidence, or of the web request, more than + * once. These tests count how often all the values of each are read in a + * call that is not shared. + */ +public class ShareUsageEvidenceReadTests { + + /** + * Enough values that copying them is most of the cost of a call. + */ + private static final int VALUE_COUNT = 100; + + /** + * A map keyed without regard to case and holding no null value, as + * Evidence and the web request are, that counts how often all its + * values are read. + */ + private static final class CountingMap + extends AbstractMap { + + private final Map values = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + private int reads; + + /** + * @return the reads since this was last called + */ + int takeReads() { + int result = reads; + reads = 0; + return result; + } + + @Override + public Set> entrySet() { + reads++; + return values.entrySet(); + } + + @Override + public Object get(Object key) { + return values.get(key); + } + + @Override + public boolean containsKey(Object key) { + return values.containsKey(key); + } + + @Override + public Object put(String key, Object value) { + return values.put(key, value); + } + + @Override + public int size() { + return values.size(); + } + } + + private static CountingMap headers() { + CountingMap headers = new CountingMap(); + for (int i = 0; i < VALUE_COUNT; i++) { + headers.put("header.x-test-" + i, "value-" + i); + } + return headers; + } + + /** + * An element with the default tracker, sharing every call that is not + * a repeat, and sending nothing until 50 calls are queued. + */ + private static ShareUsageElement createElement() throws Exception { + return new ShareUsageBuilder( + LoggerFactory.getILoggerFactory(), + mock(DataUploader.class)) + .build(); + } + + /** + * With no web request a call that is not shared reads the Evidence once, + * to make the session tracking key, and does not copy it. + */ + @Test + public void NoWebRequest_EvidenceReadOnceWhenNotShared() + throws Exception { + CountingMap evidence = headers(); + ShareUsageElement element = createElement(); + try { + FlowData data = MockFlowData.createFromEvidence(evidence, false); + element.process(data); + assertEquals(1, element.evidenceCollection.size(), + "The first call was not shared."); + + evidence.takeReads(); + element.process(data); + int reads = evidence.takeReads(); + assertEquals(1, element.evidenceCollection.size(), + "The repeat was shared."); + + assertEquals(1, reads, + "A call that is not shared read all the Evidence more than " + + "once, when the session tracking key needs it once."); + } finally { + element.close(); + } + } + + /** + * With a web request a call that is not shared reads the Evidence and + * the web request once each, to make the one copy that the session + * tracking key is made from. + */ + @Test + public void WebRequest_EachCopiedOnceWhenNotShared() throws Exception { + CountingMap request = headers(); + CountingMap evidence = new CountingMap(); + evidence.put("header.user-agent", "evidence-ua"); + evidence.put(ShareUsageEvidence.HTTP_REQUEST_KEY, request); + ShareUsageElement element = createElement(); + try { + FlowData data = MockFlowData.createFromEvidence(evidence, false); + element.process(data); + assertEquals(1, element.evidenceCollection.size(), + "The first call was not shared."); + + evidence.takeReads(); + request.takeReads(); + element.process(data); + int evidenceReads = evidence.takeReads(); + int requestReads = request.takeReads(); + assertEquals(1, element.evidenceCollection.size(), + "The repeat was shared."); + + assertEquals(1, requestReads, + "A call that is not shared copied the web request more " + + "than once."); + assertEquals(1, evidenceReads, + "A call that is not shared copied the Evidence more than " + + "once."); + } finally { + element.close(); + } + } +}