From d4a30447c42f572feeffd04c207e9f6c3417350e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 06:37:30 +0000 Subject: [PATCH 01/19] Test that usage sharing pauses sending after failed sends Usage sharing in Java has no back-off. Every batch is sent while the service is failing, and is lost. These tests expect the back-off .NET is being given, at its default settings of ten failed sends within 100 seconds and a first pause of two seconds: - after ten failed sends nothing is sent during the pause, and the evidence processed during it is all sent once the pause ends and the service answers, with no more requests processed; - closing while paused returns at once and sends nothing. They fail until the back-off is added. --- .../ShareUsageElementBackOffTests.java | 265 ++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java new file mode 100644 index 00000000..da10ee1e --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java @@ -0,0 +1,265 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.trackers.Tracker; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Usage sharing pauses sending after repeated failed sends, keeps queueing + * evidence while paused, and drains the queue once a send succeeds. These + * tests use the default settings, which pause sending for two seconds after + * ten sends fail within 100 seconds. + */ +public class ShareUsageElementBackOffTests { + + /** + * The longest a test waits for something it expects to happen. + */ + private static final long WAIT_LIMIT_MILLIS = 30000; + + private static final int DEFAULT_FAILURES = 10; + + private static final long DEFAULT_PAUSE_MILLIS = 2000; + + private static final Pattern CLIENT_IP = + Pattern.compile("([^<]*)"); + + /** + * Answers each send with the status the test sets, and records when each + * send was answered and the client IPs of the sends it accepted. + */ + private static class RecordingUploader implements DataUploader { + private final long start = System.nanoTime(); + private final List times = new ArrayList<>(); + private final List accepted = new ArrayList<>(); + private volatile int code = 500; + private ByteArrayOutputStream body; + + @Override + public synchronized OutputStream getOutputStream() { + body = new ByteArrayOutputStream(); + return body; + } + + @Override + public synchronized int getResponseCode() { + int answer = code; + times.add(now()); + if (answer == 200) { + Matcher matcher = CLIENT_IP.matcher( + new String(body.toByteArray(), StandardCharsets.UTF_8)); + while (matcher.find()) { + accepted.add(matcher.group(1)); + } + } + return answer; + } + + long now() { + return (System.nanoTime() - start) / 1000000; + } + + synchronized int count() { + return times.size(); + } + + synchronized List times() { + return new ArrayList<>(times); + } + + synchronized List accepted() { + return new ArrayList<>(accepted); + } + } + + /** + * Shares every request. + */ + private static class ShareAll implements Tracker { + @Override + public boolean track(FlowData flowData) { + return true; + } + } + + private Pipeline pipeline; + private RecordingUploader uploader; + private ShareUsageElement element; + private int requests; + + /** + * Each entry is sent on its own, as the minimum entries per message is + * one. + */ + @BeforeEach + public void init() throws Exception { + pipeline = new PipelineBuilder().build(); + uploader = new RecordingUploader(); + requests = 0; + element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + 100, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + DEFAULT_SESSION_COOKIE_NAME, + new ShareAll()); + element.addPipeline(pipeline); + element.dataUploader = uploader; + } + + @AfterEach + public void cleanUp() throws Exception { + element.close(); + pipeline.close(); + } + + /** + * Processes a request whose evidence no other request has, and returns + * its client IP. + */ + private String process() throws Exception { + int number = requests++; + String ip = "10.0." + (number / 250) + "." + (number % 250 + 1); + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_CLIENTIP_KEY, ip); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + element.process(data); + } + return ip; + } + + private static void waitFor( + BooleanSupplier condition, + Supplier message) throws InterruptedException { + long end = System.currentTimeMillis() + WAIT_LIMIT_MILLIS; + while (condition.getAsBoolean() == false) { + if (System.currentTimeMillis() > end) { + fail(message.get()); + } + Thread.sleep(20); + } + } + + /** + * Processes requests one at a time, waiting for each to be sent, so each + * is a separate failed send. + */ + private void failSends(int count) throws Exception { + for (int i = 1; i <= count; i++) { + process(); + final int expected = i; + waitFor( + () -> uploader.count() == expected && + element.isRunning() == false, + () -> "Send " + expected + " was not made."); + } + } + + /** + * Ten failed sends pause sending. Nothing is sent during the pause, the + * evidence processed during it is queued, and all of it is sent once the + * pause ends and the service answers, with no more requests processed to + * start the sending. + */ + @Test + public void ShareUsageElement_BackOff_PausesThenDrainsQueue() + throws Exception { + failSends(DEFAULT_FAILURES); + long pausedAt = uploader.now(); + + List queued = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + queued.add(process()); + } + Thread.sleep(700); + assertEquals(DEFAULT_FAILURES, uploader.count(), + "Sends were made during the pause that ten failed sends should " + + "have started."); + + uploader.code = 200; + waitFor( + () -> uploader.accepted().containsAll(queued), + () -> "The evidence processed during the pause was not all " + + "sent once the service answered. Sent: " + uploader.accepted()); + long waited = uploader.times().get(DEFAULT_FAILURES) - pausedAt; + assertTrue(waited >= DEFAULT_PAUSE_MILLIS - 500, + "Sending resumed " + waited + " ms after the pause started, " + + "before the two second pause ended."); + } + + /** + * Closing while paused returns at once rather than waiting for the pause + * to end, and sends nothing. + */ + @Test + public void ShareUsageElement_BackOff_CloseWhilePausedDoesNotWaitOrSend() + throws Exception { + failSends(DEFAULT_FAILURES); + process(); + process(); + + long start = System.currentTimeMillis(); + element.close(); + long took = System.currentTimeMillis() - start; + + assertTrue(took < 1500, + "Closing took " + took + " ms, as it waited for the pause."); + assertEquals(DEFAULT_FAILURES, uploader.count(), + "A send was made during the pause or when closing."); + } +} From b26878f087a81fa1c2afaedd83c498a526e84bbf Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 06:46:39 +0000 Subject: [PATCH 02/19] Pause usage sharing sends after repeated failures Java now has the same back-off as .NET, with the same five settings and defaults. - After setFailuresToEnterRecovery sends fail within setFailuresWindowSeconds, sending pauses for setInitialRecoveryDelaySeconds. A send fails when it throws or the answer is not 200. - When a pause ends one send is tried. If it fails the delay is multiplied by setRecoveryMultiplier, up to setMaxRecoveryDelaySeconds. If it succeeds the back-off ends. - Processing is not affected. Evidence is queued during a pause and new evidence is dropped only when the queue is full. - The consumer waits out the pause itself, on a latch that closing releases, and then sends until fewer than the minimum entries per message are queued, so the queue drains when a pause ends without more requests being processed. - Closing ends a pause at once and sends nothing while paused, logging how many entries were not sent. ShareUsageBackOff holds the rules and takes the time from the caller, so its tests set the time. The consumer calls the new protected waitWhilePaused before each message and recordSendResult after it. New constructors take the five settings, and the existing ones use the defaults. --- .../fiftyone/flowelements/Constants.java | 39 ++++ .../flowelements/ShareUsageBackOff.java | 149 ++++++++++++++ .../fiftyone/flowelements/ShareUsageBase.java | 155 +++++++++++++++ .../flowelements/ShareUsageBuilder.java | 7 +- .../flowelements/ShareUsageBuilderBase.java | 91 +++++++++ .../flowelements/ShareUsageElement.java | 111 ++++++++++- .../flowelements/ShareUsageBackOffTests.java | 184 ++++++++++++++++++ .../ShareUsageElementBackOffTests.java | 93 +++++++++ 8 files changed, 825 insertions(+), 4 deletions(-) create mode 100644 pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java index f53a7bf2..f2e536e8 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java @@ -104,4 +104,43 @@ public class Constants { */ static final String EVIDENCE_HTTPHEADER_COOKIE_SUFFIX = "cookie"; + /** + * The default number of failed sends within the failures window that + * pauses sending. + */ + public static final int SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT = 10; + + /** + * The fewest failed sends that can be set to pause sending. + */ + public static final int SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN = 1; + + /** + * The most failed sends that can be set to pause sending. + */ + public static final int SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX = 100; + + /** + * The default time in seconds over which failed sends are counted. + */ + public static final int SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT = 100; + + /** + * The default length in seconds of the first pause after failed sends. + */ + public static final double + SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT = 2.0; + + /** + * The default longest pause in seconds. + */ + public static final double + SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT = 300.0; + + /** + * The default multiplier applied to the pause each time the send tried + * when a pause ends fails. + */ + public static final double + SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT = 2.0; } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java new file mode 100644 index 00000000..3cf13cde --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java @@ -0,0 +1,149 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import java.util.ArrayDeque; +import java.util.Deque; + +/** + * Decides when usage sharing may send. Sending pauses for the initial delay + * once the set number of sends fail within the window. A failed send after a + * pause multiplies the delay, up to the maximum, and a successful send ends + * the back-off. Only sending pauses, so evidence is still queued while + * paused. Each method takes the time from the caller, in milliseconds. + */ +class ShareUsageBackOff { + + private final int failuresToEnterRecovery; + private final long failuresWindowMillis; + private final double initialDelaySeconds; + private final double maxDelaySeconds; + private final double multiplier; + + /** + * When each recent failed send happened, oldest first. Empty while + * backing off. + */ + private final Deque failures = new ArrayDeque<>(); + + /** + * The length of the current pause in seconds, or zero when not backing + * off. + */ + private double delaySeconds = 0; + + /** + * When sending may start again. + */ + private long resumeAt = Long.MIN_VALUE; + + ShareUsageBackOff( + int failuresToEnterRecovery, + int failuresWindowSeconds, + double initialDelaySeconds, + double maxDelaySeconds, + double multiplier) { + if (failuresToEnterRecovery < 1) { + throw new IllegalArgumentException( + "failuresToEnterRecovery must be 1 or more."); + } + if (failuresWindowSeconds < 1) { + throw new IllegalArgumentException( + "failuresWindowSeconds must be 1 or more."); + } + if ((initialDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "initialDelaySeconds must be more than 0."); + } + if ((maxDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "maxDelaySeconds must be more than 0."); + } + if ((multiplier >= 1) == false) { + throw new IllegalArgumentException( + "multiplier must be 1 or more."); + } + this.failuresToEnterRecovery = failuresToEnterRecovery; + this.failuresWindowMillis = failuresWindowSeconds * 1000L; + this.initialDelaySeconds = initialDelaySeconds; + this.maxDelaySeconds = maxDelaySeconds; + this.multiplier = multiplier; + } + + /** + * @return true from the start of a pause until a send succeeds + */ + synchronized boolean isBackingOff() { + return delaySeconds > 0; + } + + /** + * @param now the time now in milliseconds + * @return the milliseconds left before sending may start again, or zero + * if it may start now + */ + synchronized long remaining(long now) { + return resumeAt > now ? resumeAt - now : 0; + } + + /** + * Records a failed send. + * @param now the time now in milliseconds + * @return the new pause in milliseconds if this failure started or + * lengthened one, otherwise zero + */ + synchronized long failed(long now) { + if (delaySeconds > 0) { + // The send tried when the last pause ended failed. + delaySeconds = Math.min( + delaySeconds * multiplier, + maxDelaySeconds); + } else { + failures.addLast(now); + while (failures.peekFirst() < now - failuresWindowMillis) { + failures.removeFirst(); + } + if (failures.size() < failuresToEnterRecovery) { + return 0; + } + failures.clear(); + delaySeconds = Math.min(initialDelaySeconds, maxDelaySeconds); + } + long pause = Math.max(1, Math.round(delaySeconds * 1000)); + resumeAt = now + pause; + return pause; + } + + /** + * Records a successful send. + * @return true if this ended a back-off + */ + synchronized boolean succeeded() { + if (delaySeconds == 0) { + return false; + } + delaySeconds = 0; + resumeAt = Long.MIN_VALUE; + return true; + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index ddb8809b..c4b84e9b 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -224,6 +224,16 @@ public abstract class ShareUsageBase */ protected long lostDataReset = LOST_DATA_RESET_DEFAULT; + /** + * Decides when sending may happen after failed sends. + */ + private final ShareUsageBackOff backOff; + + /** + * Released when the element starts closing, to end any pause at once. + */ + private final CountDownLatch closing = new CountDownLatch(1); + /** * Constructor * @param logger the logger to use @@ -392,6 +402,90 @@ protected ShareUsageBase( List> ignoreDataEvidenceFilter, String sessionCookieName, Tracker tracker) { + this( + logger, + sharePercentage, + minimumEntriesPerMessage, + maximumQueueSize, + addTimeout, + takeTimeout, + repeatEvidenceIntervalMinutes, + trackSession, + shareUsageUrl, + blockedHttpHeaders, + includedQueryStringParameters, + ignoreDataEvidenceFilter, + sessionCookieName, + tracker, + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT, + Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT, + Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT, + Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, + Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT); + } + + /** + * Constructor + * @param logger the logger to use + * @param sharePercentage the approximate proportion of requests to share. + * 1 = 100%, 0.5 = 50%, etc. + * @param minimumEntriesPerMessage the minimum number of request entries per + * message sent to 51Degrees + * @param maximumQueueSize the maximum number of items to hold in the queue + * at one time. This must be larger than minimum + * entries + * @param addTimeout the timeout in milliseconds to allow when attempting to + * add an item to the queue. If this timeout is exceeded + * then the item is not shared + * @param takeTimeout the timeout in milliseconds to allow when attempting + * to take an item to the queue + * @param repeatEvidenceIntervalMinutes the interval (in minutes) which is + * used to decide if repeat evidence is + * old enough to consider a new session + * @param trackSession set if the tracker should consider sessions in share + * usage + * @param shareUsageUrl the URL to send data to + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParameters a list of the names of query string + * parameters that share usage should + * send to 51Degrees + * @param ignoreDataEvidenceFilter the filter used to determine if an item + * of evidence should be ignored or not + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param tracker the {@link Tracker} to use to determine if a given + * {@link FlowData} instance should be shared or not + * @param failuresToEnterRecovery the number of failed sends within the + * failures window that pauses sending + * @param failuresWindowSeconds the time in seconds over which failed sends + * are counted + * @param initialRecoveryDelaySeconds the length in seconds of the first + * pause + * @param maxRecoveryDelaySeconds the longest a pause can be, in seconds + * @param recoveryMultiplier what the pause is multiplied by each time the + * send tried when a pause ends fails + */ + protected ShareUsageBase( + Logger logger, + double sharePercentage, + int minimumEntriesPerMessage, + int maximumQueueSize, + int addTimeout, + int takeTimeout, + int repeatEvidenceIntervalMinutes, + boolean trackSession, + String shareUsageUrl, + List blockedHttpHeaders, + List includedQueryStringParameters, + List> ignoreDataEvidenceFilter, + String sessionCookieName, + Tracker tracker, + int failuresToEnterRecovery, + int failuresWindowSeconds, + double initialRecoveryDelaySeconds, + double maxRecoveryDelaySeconds, + double recoveryMultiplier) { super(logger, null); if (minimumEntriesPerMessage > maximumQueueSize) { @@ -406,6 +500,12 @@ protected ShareUsageBase( } evidenceCollection = new LinkedBlockingQueue<>(maximumQueueSize); + backOff = new ShareUsageBackOff( + failuresToEnterRecovery, + failuresWindowSeconds, + initialRecoveryDelaySeconds, + maxRecoveryDelaySeconds, + recoveryMultiplier); this.addTimeout = addTimeout; this.takeTimeout = takeTimeout; @@ -606,6 +706,9 @@ protected void managedResourcesCleanup() { // This is set before anything is submitted, so a consumer can never // observe it as false and then exit leaving a partial batch behind. stopping = true; + // Ends a pause at once, so closing never waits for one. Nothing is + // sent while paused. + closing.countDown(); // Queue a final consumer without consulting isRunning(). That check is // subject to a time-of-check/time-of-use race: a consumer that has // finished its loop with a sub-threshold remainder still reports as @@ -627,6 +730,13 @@ protected void managedResourcesCleanup() { // abandon its own wait immediately. logger.warn("Interrupted while awaiting close down"); } + if (evidenceCollection.isEmpty() == false && + backOff.remaining(System.currentTimeMillis()) > 0) { + logger.info( + "{} usage sharing entries were not sent, as sending was " + + "paused after failed sends when usage sharing closed.", + evidenceCollection.size()); + } } @Override @@ -784,6 +894,51 @@ private void submitSendData() { protected abstract void sendUsageData() throws HttpException; + /** + * Waits until sending may start again after failed sends. A consumer + * calls this before each message it sends. + * @return false if the element started closing while sending was paused, + * in which case nothing more should be sent + */ + protected boolean waitWhilePaused() { + long pause; + while ((pause = backOff.remaining(System.currentTimeMillis())) > 0) { + try { + if (closing.await(pause, TimeUnit.MILLISECONDS)) { + return false; + } + } catch (InterruptedException e) { + // Not re-interrupting, for the reason given in + // managedResourcesCleanup. The interrupt comes from closing. + return false; + } + } + return true; + } + + /** + * Records whether the remote service accepted a message, so that repeated + * failures pause sending and a success ends the pause. + * @param accepted true if the message was accepted + */ + protected void recordSendResult(boolean accepted) { + if (accepted) { + if (backOff.succeeded()) { + logger.info("Usage sharing resumed after a send succeeded."); + } + } else { + long pause = backOff.failed(System.currentTimeMillis()); + if (pause > 0) { + logger.warn( + "Usage sharing paused for {} seconds after failed " + + "sends. Usage data is still queued, up to the " + + "maximum queue size, and sending starts again when " + + "the pause ends.", + pause / 1000.0); + } + } + } + /** * Get engine Version No. * @return engine version no diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java index 56a96e9a..21e06859 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java @@ -129,7 +129,12 @@ public ShareUsageElement build() throws IOException { includedQueryStringParameters, ignoreDataEvidenceFilter, sessionCookieName, - null); + null, + failuresToEnterRecovery, + failuresWindowSeconds, + initialRecoveryDelaySeconds, + maxRecoveryDelaySeconds, + recoveryMultiplier); if (Objects.nonNull(this.httpClient)) { sue.httpClient = httpClient; } else if (Objects.nonNull(dataUploader)){ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index e376660c..3ad5d30f 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -59,6 +59,11 @@ public abstract class ShareUsageBuilderBase { protected final List includedQueryStringParameters = new ArrayList<>(); protected final List> ignoreDataEvidenceFilter = new ArrayList<>(); protected boolean trackSession; + protected int failuresToEnterRecovery = Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT; + protected int failuresWindowSeconds = Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT; + protected double initialRecoveryDelaySeconds = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT; + protected double maxRecoveryDelaySeconds = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; + protected double recoveryMultiplier = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; /** * Constructor @@ -420,6 +425,92 @@ public ShareUsageBuilderBase setTrackSession(boolean track) { return this; } + /** + * Set the number of failed sends within the failures window that pauses + * sending. Usage data is still queued while sending is paused, up to the + * maximum queue size. + * @param failuresToEnterRecovery the number of failed sends, from 1 to 100 + * @return this builder + */ + @DefaultValue(intValue = Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT) + public ShareUsageBuilderBase setFailuresToEnterRecovery( + int failuresToEnterRecovery) { + if (failuresToEnterRecovery < Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN || + failuresToEnterRecovery > Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX) { + throw new IllegalArgumentException( + "failuresToEnterRecovery must be from " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN + " to " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX + + ". Received: " + failuresToEnterRecovery + "."); + } + this.failuresToEnterRecovery = failuresToEnterRecovery; + return this; + } + + /** + * Set the time in seconds over which failed sends are counted. + * @param failuresWindowSeconds the time in seconds, 1 or more + * @return this builder + */ + @DefaultValue(intValue = Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT) + public ShareUsageBuilderBase setFailuresWindowSeconds( + int failuresWindowSeconds) { + if (failuresWindowSeconds < 1) { + throw new IllegalArgumentException( + "failuresWindowSeconds must be 1 or more."); + } + this.failuresWindowSeconds = failuresWindowSeconds; + return this; + } + + /** + * Set the length in seconds of the first pause after failed sends. + * @param initialDelaySeconds the length in seconds, more than 0 + * @return this builder + */ + @DefaultValue(doubleValue = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT) + public ShareUsageBuilderBase setInitialRecoveryDelaySeconds( + double initialDelaySeconds) { + if ((initialDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "initialDelaySeconds must be more than 0."); + } + this.initialRecoveryDelaySeconds = initialDelaySeconds; + return this; + } + + /** + * Set the longest a pause can be, in seconds. + * @param maxDelaySeconds the length in seconds, more than 0 + * @return this builder + */ + @DefaultValue(doubleValue = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT) + public ShareUsageBuilderBase setMaxRecoveryDelaySeconds( + double maxDelaySeconds) { + if ((maxDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "maxDelaySeconds must be more than 0."); + } + this.maxRecoveryDelaySeconds = maxDelaySeconds; + return this; + } + + /** + * Set what the pause is multiplied by each time the send tried when a + * pause ends fails. 2 doubles the pause each time. + * @param multiplier the multiplier, 1 or more + * @return this builder + */ + @DefaultValue(doubleValue = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT) + public ShareUsageBuilderBase setRecoveryMultiplier(double multiplier) { + if ((multiplier >= 1) == false) { + throw new IllegalArgumentException( + "multiplier must be 1 or more."); + } + this.recoveryMultiplier = multiplier; + return this; + } + /** * Create the {@link ShareUsageElement}. * @return the newly created target instance diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index c56e032f..79d85474 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -46,6 +46,11 @@ import java.util.zip.GZIPOutputStream; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_MAX_EVIDENCE_LENGTH; /** @@ -309,6 +314,90 @@ public class ShareUsageElement extends ShareUsageBase { List> ignoreDataEvidenceFilter, String sessionCookieName, Tracker tracker) { + this( + logger, + sharePercentage, + minimumEntriesPerMessage, + maximumQueueSize, + addTimeout, + takeTimeout, + repeatEvidenceIntervalMinutes, + trackSession, + shareUsageUrl, + blockedHttpHeaders, + includedQueryStringParameters, + ignoreDataEvidenceFilter, + sessionCookieName, + tracker, + SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT, + SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT, + SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT, + SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, + SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT); + } + + /** + * Constructor + * + * @param logger the logger to use + * @param sharePercentage the approximate proportion of requests to share. + * 1 = 100%, 0.5 = 50%, etc. + * @param minimumEntriesPerMessage the minimum number of request entries per + * message sent to 51Degrees + * @param maximumQueueSize the maximum number of items to hold in the queue + * at one time. This must be larger than minimum + * entries + * @param addTimeout the timeout in milliseconds to allow when attempting to + * add an item to the queue. If this timeout is exceeded + * then the item is not shared + * @param takeTimeout the timeout in milliseconds to allow when attempting + * to take an item to the queue + * @param repeatEvidenceIntervalMinutes the interval (in minutes) which is + * used to decide if repeat evidence is + * old enough to consider a new session + * @param trackSession set if the tracker should consider sessions in share + * usage + * @param shareUsageUrl the URL to send data to + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParameters a list of the names of query string + * parameters that share usage should + * send to 51Degrees + * @param ignoreDataEvidenceFilter the filter used to determine if an item + * of evidence should be ignored or not + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param tracker the {@link Tracker} to use to determine if a given + * {@link FlowData} instance should be shared or not + * @param failuresToEnterRecovery the number of failed sends within the failures + * window that pauses sending + * @param failuresWindowSeconds the time in seconds over which failed sends are + * counted + * @param initialRecoveryDelaySeconds the length in seconds of the first pause + * @param maxRecoveryDelaySeconds the longest a pause can be, in seconds + * @param recoveryMultiplier what the pause is multiplied by each time the send + * tried when a pause ends fails + */ + ShareUsageElement( + Logger logger, + double sharePercentage, + int minimumEntriesPerMessage, + int maximumQueueSize, + int addTimeout, + int takeTimeout, + int repeatEvidenceIntervalMinutes, + boolean trackSession, + String shareUsageUrl, + List blockedHttpHeaders, + List includedQueryStringParameters, + List> ignoreDataEvidenceFilter, + String sessionCookieName, + Tracker tracker, + int failuresToEnterRecovery, + int failuresWindowSeconds, + double initialRecoveryDelaySeconds, + double maxRecoveryDelaySeconds, + double recoveryMultiplier) { super( logger, sharePercentage, @@ -323,7 +412,12 @@ public class ShareUsageElement extends ShareUsageBase { includedQueryStringParameters, ignoreDataEvidenceFilter, sessionCookieName, - tracker); + tracker, + failuresToEnterRecovery, + failuresWindowSeconds, + initialRecoveryDelaySeconds, + maxRecoveryDelaySeconds, + recoveryMultiplier); headers.put("Content-Type", "text/xml; charset=utf-8"); // assume default GZIP connection @@ -336,7 +430,8 @@ public class ShareUsageElement extends ShareUsageBase { } /** - * Take data from the queue until there are fewer entries than the minimum batch size + * Take data from the queue until there are fewer entries than the minimum + * batch size, waiting out any pause after failed sends before each message */ @Override protected void sendUsageData() { @@ -345,6 +440,10 @@ protected void sendUsageData() { List allData = new ArrayList<>(); // drain queue while minimum entries in it do { + // Closing while paused ends the wait, and nothing more is sent. + if (waitWhilePaused() == false) { + break; + } logger.debug(threadMarker, "Queue size is {}", evidenceCollection.size()); try { allData.clear(); @@ -362,7 +461,13 @@ protected void sendUsageData() { // drained everything - and there is no point posting a document // with no entries in it. if (allData.isEmpty() == false) { - sendAsXML(allData); + boolean accepted = false; + try { + sendAsXML(allData); + accepted = true; + } finally { + recordSendResult(accepted); + } } } catch (InterruptedException interruptedException) { // Deliberately not re-interrupting: the interrupt comes from diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java new file mode 100644 index 00000000..6cb71eb7 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java @@ -0,0 +1,184 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The rules that decide when usage sharing may send after failed sends, + * with each test setting the time, and the builder settings for them. + */ +public class ShareUsageBackOffTests { + + private static final long START = 1767225600000L; + + private static long at(double seconds) { + return START + Math.round(seconds * 1000); + } + + /** + * Three failures within a ten second window start a two second pause, + * rising by two each time up to five. + */ + private static ShareUsageBackOff create( + int failuresToEnterRecovery, + double initialDelaySeconds, + double maxDelaySeconds) { + return new ShareUsageBackOff( + failuresToEnterRecovery, + 10, + initialDelaySeconds, + maxDelaySeconds, + 2); + } + + private static ShareUsageBackOff create() { + return create(3, 2, 5); + } + + @Test + public void ShareUsageBackOff_PausesAfterFailuresWithinWindow() { + ShareUsageBackOff backOff = create(); + + assertEquals(0, backOff.failed(at(0))); + assertEquals(0, backOff.failed(at(1))); + assertEquals(0, backOff.remaining(at(1)), + "Sending paused before enough sends failed."); + assertFalse(backOff.isBackingOff()); + + assertEquals(2000, backOff.failed(at(2))); + assertTrue(backOff.isBackingOff()); + assertEquals(1000, backOff.remaining(at(3))); + assertEquals(0, backOff.remaining(at(4))); + } + + @Test + public void ShareUsageBackOff_FailuresOutsideWindowDoNotCount() { + ShareUsageBackOff backOff = create(); + + backOff.failed(at(0)); + backOff.failed(at(1)); + + assertEquals(0, backOff.failed(at(11.5)), + "Failures older than the window were counted."); + assertFalse(backOff.isBackingOff()); + } + + /** + * The window counts failures, not failures in a row, so a high failure + * rate pauses sending even when some sends succeed. + */ + @Test + public void ShareUsageBackOff_SuccessesBetweenFailuresDoNotReset() { + ShareUsageBackOff backOff = create(); + + backOff.failed(at(0)); + assertFalse(backOff.succeeded()); + backOff.failed(at(1)); + assertFalse(backOff.succeeded()); + + assertEquals(2000, backOff.failed(at(2))); + } + + @Test + public void ShareUsageBackOff_FailedSendAfterPauseMultipliesDelay() { + ShareUsageBackOff backOff = create(1, 2, 5); + + assertEquals(2000, backOff.failed(at(0))); + assertEquals(4000, backOff.failed(at(2))); + assertEquals(5000, backOff.failed(at(6)), + "The delay grew past the maximum."); + assertEquals(5000, backOff.failed(at(11))); + assertEquals(5000, backOff.remaining(at(11))); + } + + @Test + public void ShareUsageBackOff_SuccessEndsBackOff() { + ShareUsageBackOff backOff = create(2, 2, 5); + backOff.failed(at(0)); + backOff.failed(at(1)); + backOff.failed(at(3)); + + assertTrue(backOff.succeeded(), + "A success did not end the back-off."); + assertFalse(backOff.isBackingOff()); + assertEquals(0, backOff.remaining(at(4))); + + // The next pause needs the full number of failures again, and starts + // from the initial delay. + assertEquals(0, backOff.failed(at(8))); + assertEquals(2000, backOff.failed(at(9))); + } + + @Test + public void ShareUsageBackOff_MaximumBelowInitialLimitsFirstPause() { + ShareUsageBackOff backOff = create(1, 10, 3); + + assertEquals(3000, backOff.failed(at(0))); + } + + @Test + public void ShareUsageBackOff_InvalidSettingsAreRejected() { + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(0, 10, 2, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 0, 2, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, 0, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, Double.NaN, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, 2, 0, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, 2, 5, 0.5)); + } + + /** + * The builder settings that pause sending after failed sends reject + * values the pause cannot work with. + */ + @Test + public void ShareUsageBuilder_RecoverySettings_RejectValuesOutOfRange() { + ShareUsageBuilder builder = + new ShareUsageBuilder(LoggerFactory.getILoggerFactory()); + + assertThrows(IllegalArgumentException.class, + () -> builder.setFailuresToEnterRecovery(0)); + assertThrows(IllegalArgumentException.class, + () -> builder.setFailuresToEnterRecovery(101)); + assertThrows(IllegalArgumentException.class, + () -> builder.setFailuresWindowSeconds(0)); + assertThrows(IllegalArgumentException.class, + () -> builder.setInitialRecoveryDelaySeconds(0)); + assertThrows(IllegalArgumentException.class, + () -> builder.setMaxRecoveryDelaySeconds(-1)); + assertThrows(IllegalArgumentException.class, + () -> builder.setRecoveryMultiplier(0.5)); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java index da10ee1e..a6a0b2e1 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java @@ -161,6 +161,40 @@ public void init() throws Exception { element.dataUploader = uploader; } + /** + * Replaces the element with one that has its own back-off settings. The + * failures window is long enough that every failure in a test counts. + */ + private void useBackOff( + int failuresToEnterRecovery, + double initialDelaySeconds, + double maxDelaySeconds, + int maximumQueueSize) throws Exception { + element.close(); + element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + maximumQueueSize, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + DEFAULT_SESSION_COOKIE_NAME, + new ShareAll(), + failuresToEnterRecovery, + 100, + initialDelaySeconds, + maxDelaySeconds, + 2); + element.addPipeline(pipeline); + element.dataUploader = uploader; + } + @AfterEach public void cleanUp() throws Exception { element.close(); @@ -262,4 +296,63 @@ public void ShareUsageElement_BackOff_CloseWhilePausedDoesNotWaitOrSend() assertEquals(DEFAULT_FAILURES, uploader.count(), "A send was made during the pause or when closing."); } + + /** + * Each failed send after a pause multiplies the delay, and the delay + * stops growing at the maximum. + */ + @Test + public void ShareUsageElement_BackOff_DelayGrowsToMaximum() + throws Exception { + useBackOff(1, 0.5, 1.0, 100); + for (int i = 0; i < 40; i++) { + process(); + } + + waitFor( + () -> uploader.count() >= 4, + () -> "Fewer than four sends were made."); + List times = uploader.times(); + long first = times.get(1) - times.get(0); + long second = times.get(2) - times.get(1); + long third = times.get(3) - times.get(2); + String shown = first + ", " + second + ", " + third + " ms"; + assertTrue(first >= 400, + "The first pause was not the initial delay. Gaps: " + shown); + assertTrue(second >= 900, + "The second pause was not doubled. Gaps: " + shown); + assertTrue(third >= 900 && third < 1900, + "The third pause was not held at the maximum. Gaps: " + shown); + } + + /** + * When the queue fills during a pause, new evidence is dropped and what + * was queued first is sent once sending resumes. + */ + @Test + public void ShareUsageElement_BackOff_FullQueueDropsNewEvidence() + throws Exception { + useBackOff(1, 2, 2, 4); + failSends(1); + + List kept = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + kept.add(process()); + } + List dropped = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + dropped.add(process()); + } + uploader.code = 200; + + waitFor( + () -> uploader.accepted().containsAll(kept), + () -> "The evidence queued during the pause was not all sent. " + + "Sent: " + uploader.accepted()); + Thread.sleep(500); + for (String ip : dropped) { + assertTrue(uploader.accepted().contains(ip) == false, + "Evidence processed when the queue was full was sent."); + } + } } From 4c53537dbef038e2016ef7c60f1525ec05c884be Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 07:12:05 +0000 Subject: [PATCH 03/19] Check the whole range of failures to enter recovery in the back-off The builder setter rejects a value outside 1 to 100, and the back-off now does too, so a class derived from ShareUsageBase gets the same check through its constructor. --- .../fiftyone/flowelements/ShareUsageBackOff.java | 10 ++++++++-- .../fiftyone/flowelements/ShareUsageBackOffTests.java | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java index 3cf13cde..8cc06753 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java @@ -63,9 +63,15 @@ class ShareUsageBackOff { double initialDelaySeconds, double maxDelaySeconds, double multiplier) { - if (failuresToEnterRecovery < 1) { + if (failuresToEnterRecovery < + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN || + failuresToEnterRecovery > + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX) { throw new IllegalArgumentException( - "failuresToEnterRecovery must be 1 or more."); + "failuresToEnterRecovery must be from " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN + + " to " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX + "."); } if (failuresWindowSeconds < 1) { throw new IllegalArgumentException( diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java index 6cb71eb7..b9716f9f 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java @@ -147,6 +147,8 @@ public void ShareUsageBackOff_MaximumBelowInitialLimitsFirstPause() { public void ShareUsageBackOff_InvalidSettingsAreRejected() { assertThrows(IllegalArgumentException.class, () -> new ShareUsageBackOff(0, 10, 2, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(101, 10, 2, 5, 2)); assertThrows(IllegalArgumentException.class, () -> new ShareUsageBackOff(1, 0, 2, 5, 2)); assertThrows(IllegalArgumentException.class, From 033155f6ca5ed16a3d23b0ed286225acc5f86f0b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 07:14:53 +0000 Subject: [PATCH 04/19] Keep history out of the comments in the changed send code The comments in the functions this change touches described what the code used to do. They now say only what the code does and why. --- .../fiftyone/flowelements/ShareUsageBase.java | 14 ++++++-------- .../fiftyone/flowelements/ShareUsageElement.java | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index c4b84e9b..f01450cb 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -709,14 +709,12 @@ protected void managedResourcesCleanup() { // Ends a pause at once, so closing never waits for one. Nothing is // sent while paused. closing.countDown(); - // Queue a final consumer without consulting isRunning(). That check is - // subject to a time-of-check/time-of-use race: a consumer that has - // finished its loop with a sub-threshold remainder still reports as - // running until its Future flips to done, which used to stop close from - // queueing the drain that would have sent the remainder. The executor is - // single threaded, so the task queued here cannot run alongside a - // consumer that is already in flight - it runs after it - and shutdown() - // still allows an already submitted task to run to completion. + // Queue a final consumer without consulting isRunning(). A consumer + // that has finished its loop with a sub-threshold remainder still + // reports as running until its Future is done, so checking would skip + // the drain that sends the remainder. The executor is single threaded, + // so the task queued here runs after any consumer already in flight, + // and shutdown() still allows an already submitted task to finish. submitSendData(); try { executor.shutdown(); diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index 79d85474..730a5970 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -447,7 +447,7 @@ protected void sendUsageData() { logger.debug(threadMarker, "Queue size is {}", evidenceCollection.size()); try { allData.clear(); - // for historical reasons max entries set to 2* + // A message holds at most twice the minimum entries. while (allData.size() < minEntriesPerMessage * 2) { ShareUsageData currentData = evidenceCollection.poll(takeTimeout, TimeUnit.MILLISECONDS); // no data available From a510270651648713bd93d936b6193dac3ff6629f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 19:50:02 +0000 Subject: [PATCH 05/19] Test the usage sharing options, the case of keys and the tracker's cache --- .../flowelements/ShareUsageOptionsTests.java | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java new file mode 100644 index 00000000..ef2e2610 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java @@ -0,0 +1,273 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.caching.Cache; +import fiftyone.caching.CacheBuilder; +import fiftyone.caching.LruPutCache; +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; +import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker; +import fiftyone.pipeline.engines.services.DataUploader; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPathFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the usage sharing options "Share all Evidence" and "Share all query + * string parameters" from a configuration file, the case of the keys the + * element reads and the number of caches a tracker holds. + * @see Specification + */ +public class ShareUsageOptionsTests { + + /** + * Keeps what the element sends so a test can read it. + */ + private static class CapturingUploader implements DataUploader { + private final ByteArrayOutputStream sent = new ByteArrayOutputStream(); + + @Override + public OutputStream getOutputStream() { + return sent; + } + + @Override + public int getResponseCode() { + return 200; + } + + Document document() throws Exception { + return DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new InputSource( + new ByteArrayInputStream(sent.toByteArray()))); + } + } + + /** + * Counts the caches the tracker asks for. + */ + private static class CountingCacheBuilder implements CacheBuilder { + final AtomicInteger built = new AtomicInteger(); + private final LruPutCache.Builder inner = new LruPutCache.Builder(); + + @Override + public Cache build(Cache c, int cacheSize) { + built.incrementAndGet(); + return inner.build(c, cacheSize); + } + } + + /** + * Options for a usage sharing element that sends every request at once, + * as a configuration file holds them. + */ + private static PipelineOptions options(String... nameAndValue) { + ElementOptions element = new ElementOptions(); + element.builderName = "ShareUsage"; + element.buildParameters.put("SharePercentage", "1"); + element.buildParameters.put("MinimumEntriesPerMessage", "1"); + element.buildParameters.put("RepeatEvidenceIntervalMinutes", "0"); + element.buildParameters.put("BlockedHttpHeader", "x-blocked"); + for (int i = 0; i < nameAndValue.length; i += 2) { + element.buildParameters.put(nameAndValue[i], nameAndValue[i + 1]); + } + PipelineOptions options = new PipelineOptions(); + options.elements.add(element); + return options; + } + + private static Map evidence() { + Map evidence = new HashMap<>(); + evidence.put("header.user-agent", "agent"); + evidence.put("header.x-blocked", "blocked"); + evidence.put("header.cookie", "a=b"); + evidence.put("cookie.plain", "cookie"); + evidence.put("session.plain", "session"); + evidence.put("query.plain", "query"); + evidence.put("query.id.email", "a@example.com"); + evidence.put("id.email", "a@example.com"); + evidence.put("id.salt", "salt"); + evidence.put("server.request-path", "/path"); + evidence.put("server.request-query", "?plain=query"); + return evidence; + } + + /** + * Builds the element from the options, gives it the Evidence and returns + * what it sends. + */ + private static Document send( + PipelineOptions options, + Map evidence) throws Exception { + CapturingUploader uploader = new CapturingUploader(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .buildFromConfiguration(options)) { + ShareUsageElement element = + pipeline.getElement(ShareUsageElement.class); + assertNotNull(element); + element.dataUploader = uploader; + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + element.process(data); + assertNotNull(element.getSendDataFuture()); + element.getSendDataFuture().get(); + } + } + return uploader.document(); + } + + private static boolean shared( + Document sent, + String category, + String name) throws Exception { + return !XPathFactory.newInstance().newXPath().evaluate( + "//Device/" + category + "[@Name='" + name + "']", + sent).isEmpty(); + } + + @Test + public void defaults_ShareOnlyWhatTheRulesAllow() throws Exception { + Document sent = send(options(), evidence()); + + assertTrue(shared(sent, "header", "user-agent")); + assertFalse(shared(sent, "header", "x-blocked")); + assertFalse(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "session", "plain")); + assertFalse(shared(sent, "query", "plain")); + } + + @Test + public void shareAllEvidence_FromConfiguration_SharesEverythingButNeverShared() + throws Exception { + Document sent = send(options("ShareAllEvidence", "true"), evidence()); + + assertTrue(shared(sent, "header", "user-agent")); + assertTrue(shared(sent, "header", "x-blocked")); + assertTrue(shared(sent, "cookie", "plain")); + assertTrue(shared(sent, "session", "plain")); + assertTrue(shared(sent, "query", "plain")); + assertFalse(shared(sent, "query", "id.email")); + assertFalse(shared(sent, "id", "email")); + assertFalse(shared(sent, "id", "salt")); + assertFalse(shared(sent, "header", "cookie")); + assertFalse(shared(sent, "server", "request-path")); + assertFalse(shared(sent, "server", "request-query")); + } + + @Test + public void shareAllEvidence_False_FromConfiguration_KeepsTheRules() + throws Exception { + Document sent = send(options("ShareAllEvidence", "false"), evidence()); + + assertFalse(shared(sent, "header", "x-blocked")); + assertFalse(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "query", "plain")); + } + + @Test + public void shareAllQueryStringParameters_FromConfiguration_SharesEveryParameter() + throws Exception { + Document sent = send( + options("ShareAllQueryStringParameters", "true"), + evidence()); + + assertTrue(shared(sent, "query", "plain")); + assertFalse(shared(sent, "query", "id.email")); + assertFalse(shared(sent, "server", "request-query")); + // The other rules still apply. + assertFalse(shared(sent, "header", "x-blocked")); + assertFalse(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "session", "plain")); + assertTrue(shared(sent, "header", "user-agent")); + } + + @Test + public void shareAllQueryStringParameters_False_FromConfiguration_KeepsTheRules() + throws Exception { + Document sent = send( + options("ShareAllQueryStringParameters", "false"), + evidence()); + + assertFalse(shared(sent, "query", "plain")); + } + + @Test + public void keysAreReadWithoutRegardToCase() throws Exception { + Map evidence = new HashMap<>(); + evidence.put("Server.Client-IP", "1.2.3.4"); + evidence.put("Query.Session-ID", "session-1"); + evidence.put("Query.Sequence", "5"); + + Document sent = send(options(), evidence); + + assertEquals("1.2.3.4", + sent.getElementsByTagName("ClientIP").item(0).getTextContent()); + assertEquals("session-1", + sent.getElementsByTagName("SessionId").item(0).getTextContent()); + assertEquals("5", + sent.getElementsByTagName("Sequence").item(0).getTextContent()); + } + + @Test + public void tracker_BuildsOneCache() { + CountingCacheBuilder builder = new CountingCacheBuilder(); + + try (ShareUsageTracker tracker = new ShareUsageTracker( + new CacheConfiguration(builder, 10), + 1000, + new EvidenceKeyFilterShareUsage( + new ArrayList(), + new ArrayList(), + false, + DEFAULT_SESSION_COOKIE_NAME))) { + assertEquals(1, builder.built.get()); + } catch (java.io.IOException e) { + throw new IllegalStateException(e); + } + } +} From f29c1a9548cea1e2d0996c6cd7999b110f7574c3 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 19:53:54 +0000 Subject: [PATCH 06/19] Add the usage sharing options to share all Evidence and all query string parameters, read keys without regard to case, and keep one tracker cache --- .../data/EvidenceKeyFilterShareUsage.java | 59 +++++++++++- .../EvidenceKeyFilterShareUsageTracker.java | 32 +++++++ .../fiftyone/flowelements/ShareUsageBase.java | 91 ++++++++++++++++++- .../flowelements/ShareUsageBuilder.java | 4 +- .../flowelements/ShareUsageBuilderBase.java | 52 +++++++++++ .../flowelements/ShareUsageElement.java | 77 +++++++++++++++- .../fiftyone/trackers/ShareUsageTracker.java | 42 ++------- .../flowelements/ShareUsageOptionsTests.java | 9 ++ .../engines/caching/DataKeyedCacheBase.java | 16 +++- 9 files changed, 336 insertions(+), 46 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 07a21856..d3c09061 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -47,9 +47,12 @@ * {@link Constants#FIFTYONE_COOKIE_PREFIX} or has been configured to be * shared using the constructor parameter. * 5) Not a header, cookie, session value or query string parameter. + * When all Evidence is shared, everything is included. When all query string + * parameters are shared, every query string parameter is included. * The prefix of a key is everything before its first '.', and the field * everything after it. An email address, the salt for hashed email - * identifiers, the query string and the path are never included. + * identifiers, the query string, the path and the Cookie header are never + * included, whatever the configuration. */ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { @@ -65,6 +68,17 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { */ private final String sessionCookieName; + /** + * If true then every key is included, apart from those that are never + * shared. + */ + private final boolean shareAllEvidence; + + /** + * If true then every query string parameter is included. + */ + private final boolean shareAllQueryStringParameters; + /** * The content of HTTP headers in this array will not be included in the * request information sent to 51degrees. Any header names added here are @@ -113,6 +127,40 @@ public EvidenceKeyFilterShareUsage( List includedQueryStringParams, boolean includeSession, String sessionCookieName) { + this( + blockedHttpHeaders, + includedQueryStringParams, + includeSession, + sessionCookieName, + false, + false); + } + + /** + * Constructor. + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParams a list of the names of query string + * parameters that share usage should send + * to 51Degrees + * @param includeSession if true then the session cookie will be included in + * the filter + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param shareAllEvidence if true then all Evidence is included, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are included + */ + public EvidenceKeyFilterShareUsage( + List blockedHttpHeaders, + List includedQueryStringParams, + boolean includeSession, + String sessionCookieName, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { + this.shareAllEvidence = shareAllEvidence; + this.shareAllQueryStringParameters = shareAllQueryStringParameters; // The Cookie header carries every cookie, including the ones the // cookie rule leaves out. this.blockedHttpHeaders.add("cookies"); @@ -144,6 +192,10 @@ public boolean include(String key) { } } + if (shareAllEvidence) { + return true; + } + boolean result; int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); if (separator > 0) { @@ -169,8 +221,9 @@ public boolean include(String key) { case EVIDENCE_QUERY_PREFIX: // Only include query string parameters that start with the // 51Degrees prefix or have been specified in the - // constructor. - result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + // constructor, unless all of them are shared. + result = shareAllQueryStringParameters || + field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || includedQueryStringParams.contains(field); break; default: diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java index 9240b326..61c2c348 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java @@ -57,6 +57,38 @@ public EvidenceKeyFilterShareUsageTracker( sessionCookieName); } + /** + * Constructor. + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParams a list of the names of query string + * parameters that share usage should send + * to 51Degrees + * @param includeSession if true then the session cookie will be included in + * the filter + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param shareAllEvidence if true then all Evidence is included, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are included + */ + public EvidenceKeyFilterShareUsageTracker( + List blockedHttpHeaders, + List includedQueryStringParams, + boolean includeSession, + String sessionCookieName, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { + super( + blockedHttpHeaders, + includedQueryStringParams, + includeSession, + sessionCookieName, + shareAllEvidence, + shareAllQueryStringParameters); + } + @Override public boolean include(String key) { if(key.equalsIgnoreCase(EVIDENCE_SESSIONID) || diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 96b24077..2470b8a9 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -396,6 +396,79 @@ protected ShareUsageBase( List> ignoreDataEvidenceFilter, String sessionCookieName, Tracker tracker) { + this( + logger, + sharePercentage, + minimumEntriesPerMessage, + maximumQueueSize, + addTimeout, + takeTimeout, + repeatEvidenceIntervalMinutes, + trackSession, + shareUsageUrl, + blockedHttpHeaders, + includedQueryStringParameters, + ignoreDataEvidenceFilter, + sessionCookieName, + tracker, + false, + false); + } + + /** + * Constructor + * @param logger the logger to use + * @param sharePercentage the approximate proportion of requests to share. + * 1 = 100%, 0.5 = 50%, etc. + * @param minimumEntriesPerMessage the minimum number of request entries per + * message sent to 51Degrees + * @param maximumQueueSize the maximum number of items to hold in the queue + * at one time. This must be larger than minimum + * entries + * @param addTimeout the timeout in milliseconds to allow when attempting to + * add an item to the queue. If this timeout is exceeded + * then usage sharing will be disabled + * @param takeTimeout the timeout in milliseconds to allow when attempting + * to take an item to the queue + * @param repeatEvidenceIntervalMinutes the interval (in minutes) which is + * used to decide if repeat evidence is + * old enough to consider a new session + * @param trackSession set if the tracker should consider sessions in share + * usage + * @param shareUsageUrl the URL to send data to + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParameters a list of the names of query string + * parameters that share usage should + * send to 51Degrees + * @param ignoreDataEvidenceFilter the filter used to determine if an item + * of evidence should be ignored or not + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param tracker the {@link Tracker} to use to determine if a given + * {@link FlowData} instance should be shared or not + * @param shareAllEvidence if true then all Evidence is shared, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are shared + */ + protected ShareUsageBase( + Logger logger, + double sharePercentage, + int minimumEntriesPerMessage, + int maximumQueueSize, + int addTimeout, + int takeTimeout, + int repeatEvidenceIntervalMinutes, + boolean trackSession, + String shareUsageUrl, + List blockedHttpHeaders, + List includedQueryStringParameters, + List> ignoreDataEvidenceFilter, + String sessionCookieName, + Tracker tracker, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { super(logger, null); if (minimumEntriesPerMessage > maximumQueueSize) { @@ -430,7 +503,12 @@ protected ShareUsageBase( includedQueryStringParameters.add(Constants.EVIDENCE_SEQUENCE_SUFIX); evidenceKeyFilterExclSession = new EvidenceKeyFilterShareUsage( - blockedHttpHeaders, includedQueryStringParameters, false, sessionCookieName); + blockedHttpHeaders, + includedQueryStringParameters, + false, + sessionCookieName, + shareAllEvidence, + shareAllQueryStringParameters); this.ignoreDataEvidenceFilter = ignoreDataEvidenceFilter; @@ -444,7 +522,12 @@ protected ShareUsageBase( cacheConfiguration, intervalMillis, new EvidenceKeyFilterShareUsageTracker( - blockedHttpHeaders, includedQueryStringParameters, trackSession, sessionCookieName)); + blockedHttpHeaders, + includedQueryStringParameters, + trackSession, + sessionCookieName, + shareAllEvidence, + shareAllQueryStringParameters)); } properties = new ArrayList<>(); @@ -711,7 +794,9 @@ private ShareUsageData getDataFromEvidence(Map found) { continue; } - switch (entry.getKey()) { + // The keys are compared without regard to case, and the constants + // are lower case. + switch (entry.getKey().toLowerCase(Locale.ROOT)) { case fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY: // The client IP is dealt with separately for backwards // compatibility purposes. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java index 56a96e9a..6e492408 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java @@ -129,7 +129,9 @@ public ShareUsageElement build() throws IOException { includedQueryStringParameters, ignoreDataEvidenceFilter, sessionCookieName, - null); + null, + shareAllEvidence, + shareAllQueryStringParameters); if (Objects.nonNull(this.httpClient)) { sue.httpClient = httpClient; } else if (Objects.nonNull(dataUploader)){ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index e376660c..c11954e4 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -59,6 +59,8 @@ public abstract class ShareUsageBuilderBase { protected final List includedQueryStringParameters = new ArrayList<>(); protected final List> ignoreDataEvidenceFilter = new ArrayList<>(); protected boolean trackSession; + protected boolean shareAllEvidence; + protected boolean shareAllQueryStringParameters; /** * Constructor @@ -143,6 +145,56 @@ public ShareUsageBuilderBase setIncludedQueryStringParameter( return this; } + /** + * If set to true, every query string and HTTP form parameter is shared, + * rather than only those that start with 51D_ or are named in the + * included parameters. The whole query string and the parameters that are + * never shared are still not shared. + *

+ * Default value is false + * @param shareAll true to share all query string parameters + * @return this builder + */ + @DefaultValue(booleanValue = false) + public ShareUsageBuilderBase setShareAllQueryStringParameters( + boolean shareAll) { + this.shareAllQueryStringParameters = shareAll; + return this; + } + + /** + * Whether every query string parameter is shared. + * @return true if all query string parameters are shared + */ + public boolean isShareAllQueryStringParameters() { + return shareAllQueryStringParameters; + } + + /** + * If set to true, all Evidence is shared, which overrides the blocked + * HTTP headers and the rules for cookies, session values and query string + * parameters. The keys that are never shared (an email address, the salt + * for hashed email identifiers, the whole query string, the path and the + * Cookie header) are still not shared. + *

+ * Default value is false + * @param shareAll true to share all Evidence + * @return this builder + */ + @DefaultValue(booleanValue = false) + public ShareUsageBuilderBase setShareAllEvidence(boolean shareAll) { + this.shareAllEvidence = shareAll; + return this; + } + + /** + * Whether all Evidence is shared. + * @return true if all Evidence is shared + */ + public boolean isShareAllEvidence() { + return shareAllEvidence; + } + /** * By default, all HTTP headers (excluding a few such as 'cookies', if they * don't start with 51D_) are shared. Individual headers can be excluded from sharing by diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index c56e032f..f680f416 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -309,6 +309,79 @@ public class ShareUsageElement extends ShareUsageBase { List> ignoreDataEvidenceFilter, String sessionCookieName, Tracker tracker) { + this( + logger, + sharePercentage, + minimumEntriesPerMessage, + maximumQueueSize, + addTimeout, + takeTimeout, + repeatEvidenceIntervalMinutes, + trackSession, + shareUsageUrl, + blockedHttpHeaders, + includedQueryStringParameters, + ignoreDataEvidenceFilter, + sessionCookieName, + tracker, + false, + false); + } + + /** + * Constructor + * @param logger the logger to use + * @param sharePercentage the approximate proportion of requests to share. + * 1 = 100%, 0.5 = 50%, etc. + * @param minimumEntriesPerMessage the minimum number of request entries per + * message sent to 51Degrees + * @param maximumQueueSize the maximum number of items to hold in the queue + * at one time. This must be larger than minimum + * entries + * @param addTimeout the timeout in milliseconds to allow when attempting to + * add an item to the queue. If this timeout is exceeded + * then usage sharing will be disabled + * @param takeTimeout the timeout in milliseconds to allow when attempting + * to take an item to the queue + * @param repeatEvidenceIntervalMinutes the interval (in minutes) which is + * used to decide if repeat evidence is + * old enough to consider a new session + * @param trackSession set if the tracker should consider sessions in share + * usage + * @param shareUsageUrl the URL to send data to + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParameters a list of the names of query string + * parameters that share usage should + * send to 51Degrees + * @param ignoreDataEvidenceFilter the filter used to determine if an item + * of evidence should be ignored or not + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param tracker the {@link Tracker} to use to determine if a given + * {@link FlowData} instance should be shared or not + * @param shareAllEvidence if true then all Evidence is shared, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are shared + */ + ShareUsageElement( + Logger logger, + double sharePercentage, + int minimumEntriesPerMessage, + int maximumQueueSize, + int addTimeout, + int takeTimeout, + int repeatEvidenceIntervalMinutes, + boolean trackSession, + String shareUsageUrl, + List blockedHttpHeaders, + List includedQueryStringParameters, + List> ignoreDataEvidenceFilter, + String sessionCookieName, + Tracker tracker, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { super( logger, sharePercentage, @@ -323,7 +396,9 @@ public class ShareUsageElement extends ShareUsageBase { includedQueryStringParameters, ignoreDataEvidenceFilter, sessionCookieName, - tracker); + tracker, + shareAllEvidence, + shareAllQueryStringParameters); headers.put("Content-Type", "text/xml; charset=utf-8"); // assume default GZIP connection diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java index c6082300..5c8f2e47 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java @@ -22,7 +22,6 @@ package fiftyone.pipeline.engines.fiftyone.trackers; -import fiftyone.caching.PutCache; import fiftyone.pipeline.core.data.DataKey; import fiftyone.pipeline.core.data.DataKeyBuilder; import fiftyone.pipeline.core.data.DataKeyBuilderDefault; @@ -32,7 +31,6 @@ import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.engines.trackers.TrackerBase; -import java.io.IOException; import java.util.Date; import java.util.Map; @@ -53,13 +51,6 @@ public class ShareUsageTracker extends TrackerBase { */ private final long interval; - /** - * The times keyed by the values usage sharing would share. The cache of - * the base class is keyed by the Evidence alone, which no longer holds - * the web request's values. - */ - private final PutCache sharedValuesCache; - /** * Construct a new instance. * @param configuration the cache configuration to use when building the @@ -75,10 +66,6 @@ public ShareUsageTracker( super(configuration); this.interval = intervalMillis; this.filter = filter; - // The base class has checked that the builder makes a PutCache. - this.sharedValuesCache = (PutCache) - configuration.getCacheBuilder().build( - null, configuration.getSize()); } @Override @@ -98,31 +85,14 @@ protected EvidenceKeyFilter getFilter() { return filter; } - @Override - public Date get(FlowData data) { - return sharedValuesCache.get(getSharedValuesKey(data)); - } - - @Override - public void put(FlowData data, Date value) { - sharedValuesCache.put(getSharedValuesKey(data), value); - } - - @Override - public void close() throws IOException { - try { - sharedValuesCache.close(); - } finally { - super.close(); - } - } - /** - * The key of the values {@link ShareUsageEvidence#find} finds that the - * filter includes, which take in the web request's values, as the - * session tracking hash is of the values that would be shared. + * The key is made from the values {@link ShareUsageEvidence#find} finds + * that the filter includes, which take in the web request's values, + * as the session tracking hash is of the values that would be shared. + * The Evidence alone does not hold them. */ - private DataKey getSharedValuesKey(FlowData data) { + @Override + protected DataKey generateKey(FlowData data) { DataKeyBuilder builder = new DataKeyBuilderDefault(); for (Map.Entry entry : ShareUsageEvidence.find(data.getEvidence()).entrySet()) { diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java index ef2e2610..65a92048 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java @@ -236,6 +236,15 @@ public void shareAllQueryStringParameters_False_FromConfiguration_KeepsTheRules( assertFalse(shared(sent, "query", "plain")); } + @Test + public void builder_ShareOptionsDefaultToFalse() { + ShareUsageBuilder builder = new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()); + + assertFalse(builder.isShareAllEvidence()); + assertFalse(builder.isShareAllQueryStringParameters()); + } + @Test public void keysAreReadWithoutRegardToCase() throws Exception { Map evidence = new HashMap<>(); diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java index fd16b2c1..348f613a 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java @@ -70,14 +70,26 @@ public DataKeyedCacheBase(CacheConfiguration configuration) { */ protected abstract EvidenceKeyFilter getFilter(); + /** + * Creates the key the internal cache holds the data of a {@link FlowData} + * under. By default this is the key the {@link FlowData} generates from + * the Evidence that {@link #getFilter()} includes. A class whose key is + * made from other values overrides this method. + * @param flowData the {@link FlowData} to make a key for + * @return the key to use in the internal cache + */ + protected DataKey generateKey(FlowData flowData) { + return flowData.generateKey(getFilter()); + } + @Override public V get(FlowData flowData) { - return internalCache.get(flowData.generateKey(getFilter())); + return internalCache.get(generateKey(flowData)); } @Override public void put(FlowData flowData, V value) { - internalCache.put(flowData.generateKey(getFilter()), value); + internalCache.put(generateKey(flowData), value); } @Override From 8a99122528b18e1697858c0d748615dcd0871cc0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:35:06 +0000 Subject: [PATCH 07/19] Test the timeouts on the legacy send --- .../ShareUsageSendTimeoutTests.java | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java new file mode 100644 index 00000000..780cb80b --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java @@ -0,0 +1,133 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A send through an {@link HttpClient} that never answers must end, or it + * would stop all sending. + */ +public class ShareUsageSendTimeoutTests { + + /** + * Opens a connection that records the timeouts set on it. + */ + private static class RecordingHttpClient implements HttpClient { + private HttpURLConnection connection; + + @Override + public HttpURLConnection connect(URL url) { + connection = new HttpURLConnection(url) { + @Override + public int getResponseCode() { + return 200; + } + + @Override + public String getResponseMessage() { + return ""; + } + + @Override + public void disconnect() { } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public void connect() { } + }; + return connection; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + return ""; + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return ""; + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return ""; + } + } + + @Test + public void ShareUsageElement_LegacySend_ConnectionHasTimeouts() + throws Exception { + RecordingHttpClient client = new RecordingHttpClient(); + ShareUsageElement element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + 100, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME, + flowData -> true); + element.httpClient = client; + try { + element.legacySendAsXML(new ArrayList<>()); + + assertEquals( + element.httpSendTimeout, + client.connection.getConnectTimeout(), + "The connection has no connect timeout."); + assertEquals( + element.httpSendTimeout, + client.connection.getReadTimeout(), + "The connection has no read timeout."); + assertTrue(element.httpSendTimeout > 0); + } finally { + element.close(); + } + } +} From 4e3838d8b45a6d59b56b68be08761298989ab93e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:35:15 +0000 Subject: [PATCH 08/19] Test the back-off time comes from the monotonic clock --- .../flowelements/ShareUsageBackOffTests.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java index b9716f9f..1a628235 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java @@ -62,6 +62,19 @@ private static ShareUsageBackOff create() { return create(3, 2, 5); } + /** + * The time the back-off is given comes from a clock that a change to the + * system time does not move. + */ + @Test + public void ShareUsageBackOff_Now_FollowsTheMonotonicClock() { + long before = System.nanoTime() / 1000000; + long now = ShareUsageBackOff.now(); + long after = System.nanoTime() / 1000000; + + assertTrue(before <= now && now <= after); + } + @Test public void ShareUsageBackOff_PausesAfterFailuresWithinWindow() { ShareUsageBackOff backOff = create(); From a801f8bb976b0efe2e782ec3df31ab32131970a8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:36:06 +0000 Subject: [PATCH 09/19] Time pauses on a monotonic clock and time out the legacy send --- .../fiftyone/flowelements/ShareUsageBackOff.java | 12 +++++++++++- .../fiftyone/flowelements/ShareUsageBase.java | 6 +++--- .../fiftyone/flowelements/ShareUsageElement.java | 3 +++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java index 8cc06753..2a582d7b 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java @@ -30,7 +30,9 @@ * once the set number of sends fail within the window. A failed send after a * pause multiplies the delay, up to the maximum, and a successful send ends * the back-off. Only sending pauses, so evidence is still queued while - * paused. Each method takes the time from the caller, in milliseconds. + * paused. Each method takes the time from the caller, in milliseconds on + * the clock that {@link #now()} reads, so a change to the system time cannot + * lengthen or skip a pause. */ class ShareUsageBackOff { @@ -57,6 +59,14 @@ class ShareUsageBackOff { */ private long resumeAt = Long.MIN_VALUE; + /** + * @return the time in milliseconds on a monotonic clock, for the caller to + * pass to the other methods. Its zero is not a calendar time. + */ + static long now() { + return System.nanoTime() / 1000000; + } + ShareUsageBackOff( int failuresToEnterRecovery, int failuresWindowSeconds, diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index d74029b2..e42d6bab 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -750,7 +750,7 @@ protected void managedResourcesCleanup() { logger.warn("Interrupted while awaiting close down"); } if (evidenceCollection.isEmpty() == false && - backOff.remaining(System.currentTimeMillis()) > 0) { + backOff.remaining(ShareUsageBackOff.now()) > 0) { logger.info( "{} usage sharing entries were not sent, as sending was " + "paused after failed sends when usage sharing closed.", @@ -925,7 +925,7 @@ private void submitSendData() { */ protected boolean waitWhilePaused() { long pause; - while ((pause = backOff.remaining(System.currentTimeMillis())) > 0) { + while ((pause = backOff.remaining(ShareUsageBackOff.now())) > 0) { try { if (closing.await(pause, TimeUnit.MILLISECONDS)) { return false; @@ -950,7 +950,7 @@ protected void recordSendResult(boolean accepted) { logger.info("Usage sharing resumed after a send succeeded."); } } else { - long pause = backOff.failed(System.currentTimeMillis()); + long pause = backOff.failed(ShareUsageBackOff.now()); if (pause > 0) { logger.warn( "Usage sharing paused for {} seconds after failed " + diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index 8033b956..6cfe7b91 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -670,6 +670,9 @@ protected void legacySendAsXML(List allData) throws Exception { } HttpURLConnection connection = httpClient.connect(new URI(shareUsageUrl.trim()).toURL()); + // A send that is never answered must end, or it stops all sending. + connection.setConnectTimeout(httpSendTimeout); + connection.setReadTimeout(httpSendTimeout); String response = httpClient.postData(connection, headers, baos.toByteArray()); int responseCode = connection.getResponseCode(); String responseMessage = connection.getResponseMessage() + "data: '" + response + "'"; From cd541e45a6ba6a3a3ea90034453fbb2ad6e0362e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:37:11 +0000 Subject: [PATCH 10/19] Test that an upload to a service that never answers times out --- .../services/DataUploaderHttpTests.java | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java new file mode 100644 index 00000000..2954f257 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java @@ -0,0 +1,84 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import static org.junit.Assert.fail; + +import java.io.OutputStream; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.util.HashMap; + +import org.junit.Test; + +/** + * An upload to a service that accepts the connection and never answers must + * end, or it would stop all usage sharing. + */ +public class DataUploaderHttpTests { + + /** + * How long the uploader waits for an answer. + */ + private static final int TIMEOUT_MILLIS = 500; + + /** + * How long the test service holds a connection open without answering. + */ + private static final int HOLD_MILLIS = 30000; + + /** + * The test fails if the upload has not ended by then. + */ + private static final int LIMIT_MILLIS = 15000; + + @Test(timeout = LIMIT_MILLIS) + public void DataUploaderHttp_NoAnswer_TimesOut() throws Exception { + try (ServerSocket server = new ServerSocket(0)) { + Thread holder = new Thread(() -> { + try (Socket socket = server.accept()) { + Thread.sleep(HOLD_MILLIS); + } catch (Exception ignored) { + // The test has ended and closed the server. + } + }); + holder.setDaemon(true); + holder.start(); + DataUploaderHttp uploader = new DataUploaderHttp( + "http://localhost:" + server.getLocalPort() + "/usage", + new HashMap<>(), + TIMEOUT_MILLIS); + try (OutputStream out = uploader.getOutputStream()) { + out.write(new byte[] {1}); + } + + try { + uploader.getResponseCode(); + fail("The upload was answered."); + } catch (SocketTimeoutException expected) { + // The wait for an answer ended. + } + } + } +} From 9eca2e4bbc32ceaf4f020fac1c496595a88ac4fb Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:37:39 +0000 Subject: [PATCH 11/19] Time out an upload that is never answered --- .../fiftyone/pipeline/engines/services/DataUploaderHttp.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java index 777584bf..0c2b044d 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java @@ -40,7 +40,8 @@ public class DataUploaderHttp implements DataUploader{ * the URL to which data is to be uploaded * @param url the URL rto which data should be uploaded * @param headers HTTP headers for the connection - * @param timeout connection timeout in milliseconds + * @param timeout milliseconds to wait to connect, and then to wait for + * an answer */ public DataUploaderHttp(String url, Map headers, int timeout) { this.url = url; @@ -55,6 +56,7 @@ public DataUploaderHttp(String url, Map headers, int timeout) { public OutputStream getOutputStream() throws Exception{ connection = (HttpURLConnection) new URI(url.trim()).toURL().openConnection(); connection.setConnectTimeout(timeout); + connection.setReadTimeout(timeout); connection.setRequestMethod("POST"); for (Map.Entry header : headers.entrySet()) { connection.setRequestProperty(header.getKey(), header.getValue()); From 80729bb61f650ffea78724a6328c7ebefeabe64b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:58:36 +0000 Subject: [PATCH 12/19] Test that discarding entries from a full queue is logged without a message per entry --- .../ShareUsageElementBackOffTests.java | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java index 41daf584..b125eea4 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java @@ -30,14 +30,18 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; import java.io.OutputStream; +import java.lang.reflect.Proxy; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.Map; import java.util.function.BooleanSupplier; import java.util.function.Supplier; @@ -129,6 +133,7 @@ public boolean track(FlowData flowData) { } private Pipeline pipeline; + private Logger logger = LoggerFactory.getLogger(ShareUsageElement.class); private RecordingUploader uploader; private ShareUsageElement element; private int requests; @@ -172,7 +177,7 @@ private void useBackOff( int maximumQueueSize) throws Exception { element.close(); element = new ShareUsageElement( - LoggerFactory.getLogger(ShareUsageElement.class), + logger, 1, 1, maximumQueueSize, @@ -357,4 +362,53 @@ public void ShareUsageElement_BackOff_FullQueueDropsNewEvidence() "Evidence processed when the queue was full was sent."); } } + + /** + * A logger that records the warnings it is given. + */ + private static Logger recordingLogger(List warnings) { + return (Logger) Proxy.newProxyInstance( + Logger.class.getClassLoader(), + new Class[] {Logger.class}, + (proxy, method, args) -> { + if (method.getName().equals("warn")) { + warnings.add(Arrays.deepToString(args)); + return null; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == String.class) { + return "test"; + } + return null; + }); + } + + /** + * Evidence discarded because the queue is full is logged, but not with a + * message for each entry discarded. + */ + @Test + public void ShareUsageElement_FullQueue_LogsWithoutAMessagePerEntry() + throws Exception { + List warnings = new CopyOnWriteArrayList<>(); + logger = recordingLogger(warnings); + useBackOff(1, 2, 2, 4); + failSends(1); + warnings.clear(); + + for (int i = 0; i < 10; i++) { + process(); + } + + long full = warnings.stream() + .filter(warning -> warning.contains("full")) + .count(); + assertTrue(full >= 1, "Discarding entries was not logged."); + assertTrue(full < 6, "A message was logged for each entry discarded."); + } } From 2ea0043cd978e3ff74e2c71070d3e9cc4297eb77 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:58:55 +0000 Subject: [PATCH 13/19] Log entries discarded from a full queue without a message per entry --- .../fiftyone/flowelements/Constants.java | 7 ++++++ .../fiftyone/flowelements/ShareUsageBase.java | 24 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java index f2e536e8..28506e77 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java @@ -93,6 +93,13 @@ public class Constants { * Default lost data count for share usage before it wraps */ static final int LOST_DATA_RESET_DEFAULT = 5000000; + + /** + * While the queue is full, a message is logged for the first entry + * discarded and then once for every this many more, never one for each + * entry. + */ + static final int QUEUE_FULL_LOG_EVERY = 1000; /** * The default URL to send usage data to. */ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index e42d6bab..9e417dcf 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -228,6 +228,13 @@ public abstract class ShareUsageBase */ protected long lostDataReset = LOST_DATA_RESET_DEFAULT; + /** + * True from the first entry discarded because the queue is full until an + * entry is next queued. While true, a message is logged only for every + * {@link Constants#QUEUE_FULL_LOG_EVERY} entries discarded. + */ + private volatile boolean queueFullReported = false; + /** * Decides when sending may happen after failed sends. */ @@ -796,6 +803,7 @@ private void processData(FlowData data, Map found) { getDataFromEvidence(found), addTimeout, TimeUnit.MILLISECONDS)) { + queueFullReported = false; logger.debug("Queued {} entries", evidenceCollection.size()); // If the collection has enough entries then start // taking data from it to be sent. @@ -803,11 +811,17 @@ private void processData(FlowData data, Map found) { trySendData(); } } else { - if (++lostData >= lostDataReset) { - logger.warn("Could not queue data for Share Usage. This could mean that" + - " the queue is too small, that the proportion of data shared is" + - " too high - or that the sending process has encountered" + - " problems sending."); + long lost = ++lostData; + if (queueFullReported == false || + lost % Constants.QUEUE_FULL_LOG_EVERY == 0) { + queueFullReported = true; + logger.warn("The usage sharing queue is full, so usage data is " + + "being discarded. This could mean that the queue is too " + + "small, that the proportion of data shared is too high, " + + "or that sending is failing. {} entries have been " + + "discarded.", lost); + } + if (lost >= lostDataReset) { lostData = 0; } } From 680ba4e1c3cca0d155163e2bbad1faac7acc146e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:59:18 +0000 Subject: [PATCH 14/19] Test that a failed answer reports the status and the content --- .../ShareUsageElementBackOffTests.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java index b125eea4..0a640555 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java @@ -51,6 +51,7 @@ import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -83,6 +84,7 @@ private static class RecordingUploader implements DataUploader { private final List times = new ArrayList<>(); private final List accepted = new ArrayList<>(); private volatile int code = 500; + private volatile String content = ""; private ByteArrayOutputStream body; @Override @@ -105,6 +107,10 @@ public synchronized int getResponseCode() { return answer; } + public String getResponseContent() { + return content; + } + long now() { return (System.nanoTime() - start) / 1000000; } @@ -411,4 +417,20 @@ public void ShareUsageElement_FullQueue_LogsWithoutAMessagePerEntry() assertTrue(full >= 1, "Discarding entries was not logged."); assertTrue(full < 6, "A message was logged for each entry discarded."); } + + /** + * A send answered with anything other than 200 reports the status code + * and the content of the answer. + */ + @Test + public void ShareUsageElement_FailedAnswer_ReportsStatusAndContent() { + uploader.content = "the service says no"; + + Exception failure = assertThrows( + Exception.class, + () -> element.sendAsXML(new ArrayList<>())); + + assertTrue(failure.getMessage().contains("500")); + assertTrue(failure.getMessage().contains("the service says no")); + } } From b81c1a1615018ec87a0d2e6721861fc0bbbe22c8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:59:36 +0000 Subject: [PATCH 15/19] Test that a failed upload gives the content of the answer --- .../services/DataUploaderHttpTests.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java index 2954f257..5948dfac 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java @@ -22,9 +22,14 @@ package fiftyone.pipeline.engines.services; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import com.sun.net.httpserver.HttpServer; + import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.net.ServerSocket; import java.net.Socket; import java.net.SocketTimeoutException; @@ -81,4 +86,33 @@ public void DataUploaderHttp_NoAnswer_TimesOut() throws Exception { } } } + + @Test(timeout = LIMIT_MILLIS) + public void DataUploaderHttp_FailedAnswer_GivesTheContent() + throws Exception { + byte[] answer = "the service says no".getBytes(StandardCharsets.UTF_8); + HttpServer server = HttpServer.create(new InetSocketAddress(0), 0); + server.createContext("/usage", exchange -> { + exchange.sendResponseHeaders(500, answer.length); + exchange.getResponseBody().write(answer); + exchange.close(); + }); + server.start(); + try { + DataUploaderHttp uploader = new DataUploaderHttp( + "http://localhost:" + server.getAddress().getPort() + "/usage", + new HashMap<>(), + TIMEOUT_MILLIS * 10); + try (OutputStream out = uploader.getOutputStream()) { + out.write(new byte[] {1}); + } + + assertEquals(500, uploader.getResponseCode()); + assertEquals( + "the service says no", + uploader.getResponseContent()); + } finally { + server.stop(0); + } + } } From aabb62dca2f85889156311d6aa9453e241c5e48f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:59:51 +0000 Subject: [PATCH 16/19] Report the status and the content of a failed answer --- .../flowelements/ShareUsageElement.java | 5 +++- .../engines/services/DataUploader.java | 7 +++++ .../engines/services/DataUploaderHttp.java | 30 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index 6cfe7b91..37a5eba7 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -509,7 +509,10 @@ protected void sendAsXML(List allData) throws Exception { // check for completion int code = dataUploader.getResponseCode(); if (code != 200) { - throw new Exception(String.format("Share Usage response code was %d ", code)); + throw new Exception(String.format( + "Share Usage response code was %d with content '%s'", + code, + dataUploader.getResponseContent())); } logger.debug(threadMarker, "Send elements done"); } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java index 38df1b00..b790a641 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java @@ -44,5 +44,12 @@ public interface DataUploader { */ int getResponseCode() throws IOException; + /** + * @return the content of the answer, which is logged when the code is + * not 200, or an empty string if the uploader cannot give it + */ + default String getResponseContent() { + return ""; + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java index 0c2b044d..9ad54e67 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java @@ -23,9 +23,11 @@ package fiftyone.pipeline.engines.services; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.zip.GZIPOutputStream; @@ -34,6 +36,11 @@ public class DataUploaderHttp implements DataUploader{ private final String url; private final Map headers; private final int timeout; + + /** + * The most characters of an answer's content that are given. + */ + private static final int LOGGED_CONTENT_LENGTH = 500; HttpURLConnection connection; /** @@ -69,5 +76,28 @@ public OutputStream getOutputStream() throws Exception{ public int getResponseCode() throws IOException { return connection.getResponseCode(); } + + /** + * Reads the content of a failed answer, as far as the most characters + * that are logged. + */ + @Override + public String getResponseContent() { + try (InputStream error = connection.getErrorStream()) { + if (error == null) { + return ""; + } + byte[] content = new byte[LOGGED_CONTENT_LENGTH]; + int length = 0; + int read; + while (length < content.length && (read = error.read( + content, length, content.length - length)) > 0) { + length += read; + } + return new String(content, 0, length, StandardCharsets.UTF_8); + } catch (IOException e) { + return ""; + } + } } From 4373aadba10b7dbec100b44eefb6976a4e0a513e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 12:29:11 +0000 Subject: [PATCH 17/19] Guard the failure deque against unboxing an empty first entry --- .../engines/fiftyone/flowelements/ShareUsageBackOff.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java index 2a582d7b..d0bb4d0c 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java @@ -136,7 +136,11 @@ synchronized long failed(long now) { maxDelaySeconds); } else { failures.addLast(now); - while (failures.peekFirst() < now - failuresWindowMillis) { + // The emptiness check comes first so that nothing is unboxed + // from an empty deque. The constructor refuses a window below + // one second, so the deque always keeps the failure just added. + while (failures.isEmpty() == false && + failures.peekFirst() < now - failuresWindowMillis) { failures.removeFirst(); } if (failures.size() < failuresToEnterRecovery) { From e04b9617113bbf2e80a83e4a44ac8acd45fc6893 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 13:52:02 +0000 Subject: [PATCH 18/19] Let the upload tests keep restricted headers allowed for the run --- jdkhttp.java | 3779 +++++++++++++++++ .../services/DataUploaderHttpTests.java | 14 + 2 files changed, 3793 insertions(+) create mode 100644 jdkhttp.java diff --git a/jdkhttp.java b/jdkhttp.java new file mode 100644 index 00000000..7b750642 --- /dev/null +++ b/jdkhttp.java @@ -0,0 +1,3779 @@ +/* + * Copyright (c) 1995, 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package sun.net.www.protocol.http; + +import java.util.Arrays; +import java.net.URL; +import java.net.URLConnection; +import java.net.ProtocolException; +import java.net.HttpRetryException; +import java.net.PasswordAuthentication; +import java.net.Authenticator; +import java.net.HttpCookie; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.net.SocketTimeoutException; +import java.net.Proxy; +import java.net.ProxySelector; +import java.net.URI; +import java.net.InetSocketAddress; +import java.net.CookieHandler; +import java.net.ResponseCache; +import java.net.CacheResponse; +import java.net.SecureCacheResponse; +import java.net.CacheRequest; +import java.net.Authenticator.RequestorType; +import java.io.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Map; +import java.util.List; +import java.util.Locale; +import java.util.StringTokenizer; +import java.util.Iterator; +import java.util.HashSet; +import java.util.HashMap; +import java.util.Set; +import java.util.StringJoiner; +import jdk.internal.access.JavaNetHttpCookieAccess; +import jdk.internal.access.SharedSecrets; +import jdk.internal.util.Exceptions; +import sun.net.NetProperties; +import sun.net.NetworkClient; +import sun.net.util.IPAddressUtil; +import sun.net.www.*; +import sun.net.www.http.HttpClient; +import sun.net.www.http.PosterOutputStream; +import sun.net.www.http.ChunkedInputStream; +import sun.net.www.http.ChunkedOutputStream; +import sun.util.logging.PlatformLogger; +import java.text.SimpleDateFormat; +import java.util.TimeZone; +import java.net.MalformedURLException; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.concurrent.locks.ReentrantLock; + +import static sun.net.util.ProxyUtil.copyProxy; +import static sun.net.www.protocol.http.AuthScheme.BASIC; +import static sun.net.www.protocol.http.AuthScheme.DIGEST; +import static sun.net.www.protocol.http.AuthScheme.NTLM; +import static sun.net.www.protocol.http.AuthScheme.NEGOTIATE; +import static sun.net.www.protocol.http.AuthScheme.KERBEROS; +import static sun.net.www.protocol.http.AuthScheme.UNKNOWN; + +/** + * A class to represent an HTTP connection to a remote object. + */ + + +public class HttpURLConnection extends java.net.HttpURLConnection { + + static final String HTTP_CONNECT = "CONNECT"; + + static final String version; + public static final String userAgent; + + /* max # of allowed re-directs */ + static final int defaultmaxRedirects = 20; + static final int maxRedirects; + + /* Not all servers support the (Proxy)-Authentication-Info headers. + * By default, we don't require them to be sent + */ + static final boolean validateProxy; + static final boolean validateServer; + + /** A, possibly empty, set of authentication schemes that are disabled + * when proxying plain HTTP ( not HTTPS ). */ + static final Set disabledProxyingSchemes; + + /** A, possibly empty, set of authentication schemes that are disabled + * when setting up a tunnel for HTTPS ( HTTP CONNECT ). */ + static final Set disabledTunnelingSchemes; + + private StreamingOutputStream strOutputStream; + private static final String RETRY_MSG1 = + "cannot retry due to proxy authentication, in streaming mode"; + private static final String RETRY_MSG2 = + "cannot retry due to server authentication, in streaming mode"; + private static final String RETRY_MSG3 = + "cannot retry due to redirection, in streaming mode"; + + /* + * System properties related to error stream handling: + * + * sun.net.http.errorstream.enableBuffering = + * + * With the above system property set to true (default is false), + * when the response code is >=400, the HTTP handler will try to + * buffer the response body (up to a certain amount and within a + * time limit). Thus freeing up the underlying socket connection + * for reuse. The rationale behind this is that usually when the + * server responds with a >=400 error (client error or server + * error, such as 404 file not found), the server will send a + * small response body to explain who to contact and what to do to + * recover. With this property set to true, even if the + * application doesn't call getErrorStream(), read the response + * body, and then call close(), the underlying socket connection + * can still be kept-alive and reused. The following two system + * properties provide further control to the error stream + * buffering behaviour. + * + * sun.net.http.errorstream.timeout = + * the timeout (in millisec) waiting the error stream + * to be buffered; default is 300 ms + * + * sun.net.http.errorstream.bufferSize = + * the size (in bytes) to use for the buffering the error stream; + * default is 4k + */ + + + /* Should we enable buffering of error streams? */ + private static final boolean enableESBuffer; + + /* timeout waiting for read for buffered error stream; + */ + private static final int timeout4ESBuffer; + + /* buffer size for buffered error stream; + */ + private static final int bufSize4ES; + + private static final int maxHeaderSize; + + /* + * Restrict setting of request headers through the public api + * consistent with JavaScript XMLHttpRequest2 with a few + * exceptions. Disallowed headers are silently ignored for + * backwards compatibility reasons. For example, some applets set the + * Host header since old JREs did not implement HTTP 1.1. + * Additionally, any header starting with Sec- is + * disallowed. + * + * The following headers are allowed for historical reasons: + * + * Accept-Charset, Accept-Encoding, Cookie, Cookie2, Date, + * Referer, TE, User-Agent, headers beginning with Proxy-. + * + * The following headers are allowed in a limited form: + * + * Connection: close + * + * See http://www.w3.org/TR/XMLHttpRequest2. + */ + private static final boolean allowRestrictedHeaders; + private static final Set restrictedHeaderSet; + private static final String[] restrictedHeaders = { + /* Restricted by XMLHttpRequest2 */ + //"Accept-Charset", + //"Accept-Encoding", + "Access-Control-Request-Headers", + "Access-Control-Request-Method", + "Connection", /* close is allowed */ + "Content-Length", + //"Cookie", + //"Cookie2", + "Content-Transfer-Encoding", + //"Date", + //"Expect", + "Host", + "Keep-Alive", + "Origin", + // "Referer", + // "TE", + "Trailer", + "Transfer-Encoding", + "Upgrade", + //"User-Agent", + "Via" + }; + + private static Set schemesListToSet(String list) { + if (list == null || list.isEmpty()) + return Collections.emptySet(); + + Set s = new HashSet<>(); + String[] parts = list.split("\\s*,\\s*"); + for (String part : parts) + s.add(part.toLowerCase(Locale.ROOT)); + return s; + } + + static { + maxRedirects = Integer.getInteger("http.maxRedirects", defaultmaxRedirects); + version = System.getProperty("java.version"); + String agent = System.getProperty("http.agent"); + if (agent == null) { + agent = "Java/"+version; + } else { + agent = agent + " Java/"+version; + } + userAgent = agent; + + // A set of net properties to control the use of authentication schemes + // when proxying/tunneling. + String p = NetProperties.get("jdk.http.auth.tunneling.disabledSchemes"); + disabledTunnelingSchemes = schemesListToSet(p); + p = NetProperties.get("jdk.http.auth.proxying.disabledSchemes"); + disabledProxyingSchemes = schemesListToSet(p); + + validateProxy = Boolean.getBoolean("http.auth.digest.validateProxy"); + validateServer = Boolean.getBoolean("http.auth.digest.validateServer"); + + enableESBuffer = Boolean.getBoolean("sun.net.http.errorstream.enableBuffering"); + int esBufferTimeout = Integer.getInteger( + "sun.net.http.errorstream.timeout", 300); + if (esBufferTimeout <= 0) { + esBufferTimeout = 300; // use the default + } + timeout4ESBuffer = esBufferTimeout; + + int esBufSize = Integer.getInteger( + "sun.net.http.errorstream.bufferSize", 4096); + if (esBufSize <= 0) { + esBufSize = 4096; // use the default + } + bufSize4ES = esBufSize; + + allowRestrictedHeaders = Boolean.getBoolean("sun.net.http.allowRestrictedHeaders"); + if (!allowRestrictedHeaders) { + restrictedHeaderSet = HashSet.newHashSet(restrictedHeaders.length); + for (int i=0; i < restrictedHeaders.length; i++) { + restrictedHeaderSet.add(restrictedHeaders[i].toLowerCase(Locale.ROOT)); + } + } else { + restrictedHeaderSet = null; + } + + int defMaxHeaderSize = 384 * 1024; + String maxHeaderSizeStr = NetProperties.get("jdk.http.maxHeaderSize"); + int maxHeaderSizeVal = defMaxHeaderSize; + if (maxHeaderSizeStr != null) { + try { + maxHeaderSizeVal = Integer.parseInt(maxHeaderSizeStr); + } catch (NumberFormatException n) { + maxHeaderSizeVal = defMaxHeaderSize; + } + } + if (maxHeaderSizeVal < 0) maxHeaderSizeVal = 0; + maxHeaderSize = maxHeaderSizeVal; + } + + static final String httpVersion = "HTTP/1.1"; + static final String acceptString = "*/*"; + + // the following http request headers should NOT have their values + // returned for security reasons. + private static final String[] EXCLUDE_HEADERS = { + "Proxy-Authorization", + "Authorization" + }; + + // also exclude system cookies when any might be set + private static final String[] EXCLUDE_HEADERS2= { + "Proxy-Authorization", + "Authorization", + "Cookie", + "Cookie2" + }; + + protected HttpClient http; + protected Handler handler; + protected Proxy instProxy; + protected volatile Authenticator authenticator; + protected volatile AuthCacheImpl authCache = AuthCacheImpl.getDefault(); + + private CookieHandler cookieHandler; + private final ResponseCache cacheHandler; + + private volatile boolean usingProxy; + + // the cached response, and cached response headers and body + protected CacheResponse cachedResponse; + private MessageHeader cachedHeaders; + private InputStream cachedInputStream; + + /* output stream to server */ + protected PrintStream ps = null; + + /* buffered error stream */ + private InputStream errorStream = null; + + /* User set Cookies */ + private boolean setUserCookies = true; + private String userCookies = null; + private String userCookies2 = null; + + /* all the headers we send + * NOTE: do *NOT* dump out the content of 'requests' in the + * output or stacktrace since it may contain security-sensitive + * headers such as those defined in EXCLUDE_HEADERS. + */ + private MessageHeader requests; + + /* The headers actually set by the user are recorded here also + */ + private final MessageHeader userHeaders; + + /* Headers and request method cannot be changed + * once this flag is set in :- + * - getOutputStream() + * - getInputStream()) + * - connect() + * Access is protected by connectionLock. + */ + private boolean connecting = false; + + /* The following two fields are only used with Digest Authentication */ + String domain; /* The list of authentication domains */ + DigestAuthentication.Parameters digestparams; + + /* Current credentials in use */ + AuthenticationInfo currentProxyCredentials = null; + AuthenticationInfo currentServerCredentials = null; + boolean needToCheck = true; + private boolean doingNTLM2ndStage = false; /* doing the 2nd stage of an NTLM server authentication */ + private boolean doingNTLMp2ndStage = false; /* doing the 2nd stage of an NTLM proxy authentication */ + + /* try auth without calling Authenticator. Used for transparent NTLM authentication */ + private boolean tryTransparentNTLMServer = true; + private boolean tryTransparentNTLMProxy = true; + private boolean useProxyResponseCode = false; + + // used when redirecting to compare current and previous proxies + private Proxy lastProxy; + + /* Used by Windows specific code */ + private Object authObj; + + /* Set if the user is manually setting the Authorization or Proxy-Authorization headers */ + boolean isUserServerAuth; + boolean isUserProxyAuth; + + String serverAuthKey, proxyAuthKey; + + /* all the response headers we get back */ + private MessageHeader responses; + /* the stream _from_ the server */ + private InputStream inputStream = null; + /* post stream _to_ the server, if any */ + private PosterOutputStream poster = null; + + /* Indicates if the std. request headers have been set in requests. */ + private boolean setRequests=false; + + /* Indicates whether a request has already failed or not */ + private boolean failedOnce=false; + + /* Remembered Exception, we will throw it again if somebody + calls getInputStream after disconnect */ + private Exception rememberedException = null; + + /* Remembered Exception, we will throw it again if somebody + calls getOutputStream after disconnect or error */ + private Exception rememberedExceptionOut = null; + + /* If we decide we want to reuse a client, we put it here */ + private HttpClient reuseClient = null; + + /* Tunnel states */ + public enum TunnelState { + /* No tunnel */ + NONE, + + /* Setting up a tunnel */ + SETUP, + + /* Tunnel has been successfully setup */ + TUNNELING + } + + private TunnelState tunnelState = TunnelState.NONE; + + /* Redefine timeouts from java.net.URLConnection as we need -1 to mean + * not set. This is to ensure backward compatibility. + */ + private int connectTimeout = NetworkClient.DEFAULT_CONNECT_TIMEOUT; + private int readTimeout = NetworkClient.DEFAULT_READ_TIMEOUT; + + /* Logging support */ + private static final PlatformLogger logger = + PlatformLogger.getLogger("sun.net.www.protocol.http.HttpURLConnection"); + + /* Lock */ + private final ReentrantLock connectionLock = new ReentrantLock(); + + private final void lock() { + connectionLock.lock(); + } + + private final void unlock() { + connectionLock.unlock(); + } + + public final boolean isLockHeldByCurrentThread() { + return connectionLock.isHeldByCurrentThread(); + } + + /* + * Calls Authenticator.requestPasswordAuthentication + * + */ + private static PasswordAuthentication requestPassword( + final Authenticator authenticator, + final String host, + final InetAddress addr, + final int port, + final String protocol, + final String prompt, + final String scheme, + final URL url, + final RequestorType authType) { + + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + logger.finest("Requesting Authentication: host =" + host + " url = " + url); + } + PasswordAuthentication pass = Authenticator.requestPasswordAuthentication( + authenticator, host, addr, port, protocol, + prompt, scheme, url, authType); + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + logger.finest("Authentication returned: " + (pass != null ? pass.toString() : "null")); + } + return pass; + } + + private boolean isRestrictedHeader(String key, String value) { + if (allowRestrictedHeaders) { + return false; + } + + key = key.toLowerCase(Locale.ROOT); + if (restrictedHeaderSet.contains(key)) { + /* + * Exceptions to restricted headers: + * + * Allow "Connection: close". + */ + if (key.equals("connection") && value.equalsIgnoreCase("close")) { + return false; + } + return true; + } else if (key.startsWith("sec-")) { + return true; + } + return false; + } + + /* + * Checks the validity of http message header and whether the header + * is restricted and throws IllegalArgumentException if invalid or + * restricted. + */ + private boolean isExternalMessageHeaderAllowed(String key, String value) { + checkMessageHeader(key, value); + if (!isRestrictedHeader(key, value)) { + return true; + } + return false; + } + + /* Logging support */ + public static PlatformLogger getHttpLogger() { + return logger; + } + + /* Used for Windows NTLM implementation */ + public Object authObj() { + return authObj; + } + + public void authObj(Object authObj) { + this.authObj = authObj; + } + + @Override + public void setAuthenticator(Authenticator auth) { + lock(); + try { + if (connecting || connected) { + throw new IllegalStateException( + "Authenticator must be set before connecting"); + } + authenticator = Objects.requireNonNull(auth); + authCache = AuthCacheImpl.getAuthCacheFor(authenticator); + } finally { + unlock(); + } + } + + public AuthCacheImpl getAuthCache() { + return authCache; + } + + /* + * checks the validity of http message header and throws + * IllegalArgumentException if invalid. + */ + private void checkMessageHeader(String key, String value) { + char LF = '\n'; + int index = key.indexOf(LF); + int index1 = key.indexOf(':'); + if (index != -1 || index1 != -1) { + throw new IllegalArgumentException( + "Illegal character(s) in message header field: " + key); + } + else { + if (value == null) { + return; + } + + index = value.indexOf(LF); + while (index != -1) { + index++; + if (index < value.length()) { + char c = value.charAt(index); + if ((c==' ') || (c=='\t')) { + // ok, check the next occurrence + index = value.indexOf(LF, index); + continue; + } + } + throw new IllegalArgumentException( + "Illegal character(s) in message header value: " + value); + } + } + } + + public void setRequestMethod(String method) + throws ProtocolException { + lock(); + try { + if (connected || connecting) { + throw new IllegalStateException("Already connected"); + } + super.setRequestMethod(method); + } finally { + unlock(); + } + } + + /* adds the standard key/val pairs to requests if necessary & write to + * given PrintStream + */ + private void writeRequests() throws IOException { + assert isLockHeldByCurrentThread(); + + /* print all message headers in the MessageHeader + * onto the wire - all the ones we've set and any + * others that have been set + */ + // send any pre-emptive authentication + if (http.usingProxy && tunnelState() != TunnelState.TUNNELING) { + setPreemptiveProxyAuthentication(requests); + } + if (!setRequests) { + + /* We're very particular about the order in which we + * set the request headers here. The order should not + * matter, but some careless CGI programs have been + * written to expect a very particular order of the + * standard headers. To name names, the order in which + * Navigator3.0 sends them. In particular, we make *sure* + * to send Content-type: <> and Content-length:<> second + * to last and last, respectively, in the case of a POST + * request. + */ + final String requestLine = method + " " + getRequestURI()+ " " + httpVersion; + final int requestLineIndex = requests.getKey(requestLine); + if (requestLineIndex != 0) { + // we expect the request line to be at index 0. we set it here + // if we don't find the request line at that index. + requests.prepend(requestLine, null); + } + if (!getUseCaches()) { + requests.setIfNotSet ("Cache-Control", "no-cache"); + requests.setIfNotSet ("Pragma", "no-cache"); + } + requests.setIfNotSet("User-Agent", userAgent); + int port = url.getPort(); + String host = stripIPv6ZoneId(url.getHost()); + if (port != -1 && port != url.getDefaultPort()) { + host += ":" + String.valueOf(port); + } + // if the "Host" header hasn't been explicitly set, then set its + // value to the one determined through the request URL + requests.setIfNotSet("Host", host); + + requests.setIfNotSet("Accept", acceptString); + + /* + * For HTTP/1.1 the default behavior is to keep connections alive. + * However, we may be talking to a 1.0 server so we should set + * keep-alive just in case, except if we have encountered an error + * or if keep alive is disabled via a system property + */ + + if (http.getHttpKeepAliveSet()) { + if (http.usingProxy && tunnelState() != TunnelState.TUNNELING) { + requests.setIfNotSet("Proxy-Connection", "keep-alive"); + } else { + requests.setIfNotSet("Connection", "keep-alive"); + } + } else { + /* + * RFC 2616 HTTP/1.1 section 14.10 says: + * HTTP/1.1 applications that do not support persistent + * connections MUST include the "close" connection option + * in every message + */ + requests.setIfNotSet("Connection", "close"); + } + // Set modified since if necessary + long modTime = getIfModifiedSince(); + if (modTime != 0 ) { + Date date = new Date(modTime); + //use the preferred date format according to RFC 2068(HTTP1.1), + // RFC 822 and RFC 1123 + SimpleDateFormat fo = + new SimpleDateFormat ("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US); + fo.setTimeZone(TimeZone.getTimeZone("GMT")); + requests.setIfNotSet("If-Modified-Since", fo.format(date)); + } + // check for preemptive authorization + AuthenticationInfo sauth = AuthenticationInfo.getServerAuth(url, authCache); + if (sauth != null && sauth.supportsPreemptiveAuthorization() ) { + // Sets "Authorization" + requests.setIfNotSet(sauth.getHeaderName(), sauth.getHeaderValue(url,method)); + currentServerCredentials = sauth; + } + + if (!method.equals("PUT") && (poster != null || streaming())) { + requests.setIfNotSet ("Content-type", + "application/x-www-form-urlencoded"); + } + + boolean chunked = false; + + if (streaming()) { + if (chunkLength != -1) { + requests.set ("Transfer-Encoding", "chunked"); + chunked = true; + } else { /* fixed content length */ + if (fixedContentLengthLong != -1) { + requests.set ("Content-Length", + String.valueOf(fixedContentLengthLong)); + } else if (fixedContentLength != -1) { + requests.set ("Content-Length", + String.valueOf(fixedContentLength)); + } + } + } else if (poster != null) { + /* add Content-Length & POST/PUT data */ + // safe to synchronize on poster: this is + // a simple subclass of ByteArrayOutputStream + synchronized (poster) { + /* close it, so no more data can be added */ + poster.close(); + requests.set("Content-Length", + String.valueOf(poster.size())); + } + } + + if (!chunked) { + if (requests.findValue("Transfer-Encoding") != null) { + requests.remove("Transfer-Encoding"); + if (logger.isLoggable(PlatformLogger.Level.WARNING)) { + logger.warning( + "use streaming mode for chunked encoding"); + } + } + } + + // get applicable cookies based on the uri and request headers + // add them to the existing request headers + setCookieHeader(); + + setRequests=true; + } + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine(requests.toString()); + } + http.writeRequests(requests, poster, streaming()); + if (ps.checkError()) { + String proxyHost = http.getProxyHostUsed(); + int proxyPort = http.getProxyPortUsed(); + disconnectInternal(); + if (failedOnce) { + throw new IOException("Error writing to server"); + } else { // try once more + failedOnce=true; + if (proxyHost != null) { + setProxiedClient(url, proxyHost, proxyPort); + } else { + setNewClient (url); + } + ps = (PrintStream) http.getOutputStream(); + connected=true; + responses = new MessageHeader(maxHeaderSize); + setRequests=false; + writeRequests(); + } + } + } + + /** + * Create a new HttpClient object, bypassing the cache of + * HTTP client objects/connections. + * + * @param url the URL being accessed + */ + protected void setNewClient (URL url) + throws IOException { + setNewClient(url, false); + } + + /** + * Obtain a HttpsClient object. Use the cached copy if specified. + * + * @param url the URL being accessed + * @param useCache whether the cached connection should be used + * if present + */ + protected void setNewClient (URL url, boolean useCache) + throws IOException { + http = HttpClient.New(url, null, -1, useCache, connectTimeout, this); + http.setReadTimeout(readTimeout); + } + + + /** + * Create a new HttpClient object, set up so that it uses + * per-instance proxying to the given HTTP proxy. This + * bypasses the cache of HTTP client objects/connections. + * + * @param url the URL being accessed + * @param proxyHost the proxy host to use + * @param proxyPort the proxy port to use + */ + protected void setProxiedClient (URL url, String proxyHost, int proxyPort) + throws IOException { + setProxiedClient(url, proxyHost, proxyPort, false); + } + + /** + * Obtain a HttpClient object, set up so that it uses per-instance + * proxying to the given HTTP proxy. Use the cached copy of HTTP + * client objects/connections if specified. + * + * @param url the URL being accessed + * @param proxyHost the proxy host to use + * @param proxyPort the proxy port to use + * @param useCache whether the cached connection should be used + * if present + */ + protected void setProxiedClient (URL url, + String proxyHost, int proxyPort, + boolean useCache) + throws IOException { + proxiedConnect(url, proxyHost, proxyPort, useCache); + } + + protected void proxiedConnect(URL url, + String proxyHost, int proxyPort, + boolean useCache) + throws IOException { + http = HttpClient.New (url, proxyHost, proxyPort, useCache, + connectTimeout, this); + http.setReadTimeout(readTimeout); + } + + protected HttpURLConnection(URL u, Handler handler) + throws IOException { + // we set proxy == null to distinguish this case with the case + // when per connection proxy is set + this(u, null, handler); + } + + private static String checkHost(String h) throws IOException { + if (h != null) { + if (h.indexOf('\n') > -1) { + throw new MalformedURLException("Illegal character in host"); + } + } + return h; + } + public HttpURLConnection(URL u, String host, int port) throws IOException { + this(u, new Proxy(Proxy.Type.HTTP, + InetSocketAddress.createUnresolved(checkHost(host), port))); + } + + /** this constructor is used by other protocol handlers such as ftp + that want to use http to fetch urls on their behalf.*/ + public HttpURLConnection(URL u, Proxy p) throws IOException { + this(u, p, new Handler()); + } + + private static URL checkURL(URL u) throws IOException { + if (u != null) { + if (u.toExternalForm().indexOf('\n') > -1) { + throw new MalformedURLException("Illegal character in URL"); + } + } + String s = IPAddressUtil.checkAuthority(u); + if (s != null) { + throw new MalformedURLException(s); + } + return u; + } + + protected HttpURLConnection(URL u, Proxy p, Handler handler) + throws IOException { + super(checkURL(u)); + requests = new MessageHeader(); + responses = new MessageHeader(maxHeaderSize); + userHeaders = new MessageHeader(); + this.handler = handler; + instProxy = copyProxy(p); + cookieHandler = CookieHandler.getDefault(); + cacheHandler = ResponseCache.getDefault(); + } + + // overridden in HTTPS subclass + + public void connect() throws IOException { + lock(); + try { + connecting = true; + } finally { + unlock(); + } + plainConnect(); + } + + private boolean checkReuseConnection () { + if (connected) { + return true; + } + if (reuseClient != null) { + http = reuseClient; + http.setReadTimeout(getReadTimeout()); + http.reuse = false; + reuseClient = null; + connected = true; + return true; + } + return false; + } + + protected void plainConnect() throws IOException { + lock(); + try { + if (connected) { + return; + } + } finally { + unlock(); + } + plainConnect0(); + } + + protected void plainConnect0() throws IOException { + // try to see if request can be served from local cache + if (cacheHandler != null && getUseCaches()) { + try { + URI uri = ParseUtil.toURI(url); + if (uri != null) { + cachedResponse = cacheHandler.get(uri, getRequestMethod(), getUserSetHeaders().getHeaders()); + if ("https".equalsIgnoreCase(uri.getScheme()) + && !(cachedResponse instanceof SecureCacheResponse)) { + cachedResponse = null; + } + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + logger.finest("Cache Request for " + uri + " / " + getRequestMethod()); + logger.finest("From cache: " + (cachedResponse != null ? cachedResponse.toString() : "null")); + } + if (cachedResponse != null) { + cachedHeaders = mapToMessageHeader(cachedResponse.getHeaders()); + cachedInputStream = cachedResponse.getBody(); + } + } + } catch (IOException ioex) { + // ignore and commence normal connection + } + if (cachedHeaders != null && cachedInputStream != null) { + connected = true; + return; + } else { + cachedResponse = null; + } + } + try { + /* Try to open connections using the following scheme, + * return on the first one that's successful: + * 1) if (instProxy != null) + * connect to instProxy; raise exception if failed + * 2) else use system default ProxySelector + * 3) else make a direct connection if ProxySelector is not present + */ + + if (instProxy == null) { // no instance Proxy is set + /** + * Do we have to use a proxy? + */ + final ProxySelector sel = ProxySelector.getDefault(); + if (sel != null) { + URI uri = sun.net.www.ParseUtil.toURI(url); + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + logger.finest("ProxySelector Request for " + uri); + } + final List proxies; + try { + proxies = sel.select(uri); + } catch (IllegalArgumentException iae) { + throw new IOException("Failed to select a proxy", iae); + } + final Iterator it = proxies.iterator(); + Proxy p; + while (it.hasNext()) { + p = copyProxy(it.next()); + try { + if (!failedOnce) { + http = getNewHttpClient(url, p, connectTimeout); + http.setReadTimeout(readTimeout); + } else { + // make sure to construct new connection if first + // attempt failed + http = getNewHttpClient(url, p, connectTimeout, false); + http.setReadTimeout(readTimeout); + } + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + if (p != null) { + logger.finest("Proxy used: " + p.toString()); + } + } + break; + } catch (IOException ioex) { + if (p != Proxy.NO_PROXY) { + sel.connectFailed(uri, p.address(), ioex); + if (!it.hasNext()) { + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + logger.finest("Retrying with proxy: " + p.toString()); + } + http = getNewHttpClient(url, p, connectTimeout, false); + http.setReadTimeout(readTimeout); + break; + } + } else { + throw ioex; + } + continue; + } + } + } else { + // No proxy selector, create http client with no proxy + if (!failedOnce) { + http = getNewHttpClient(url, null, connectTimeout); + http.setReadTimeout(readTimeout); + } else { + // make sure to construct new connection if first + // attempt failed + http = getNewHttpClient(url, null, connectTimeout, false); + http.setReadTimeout(readTimeout); + } + } + } else { + if (!failedOnce) { + http = getNewHttpClient(url, instProxy, connectTimeout); + http.setReadTimeout(readTimeout); + } else { + // make sure to construct new connection if first + // attempt failed + http = getNewHttpClient(url, instProxy, connectTimeout, false); + http.setReadTimeout(readTimeout); + } + } + + usingProxy = usingProxy || usingProxyInternal(); + ps = (PrintStream)http.getOutputStream(); + } catch (IOException e) { + throw e; + } + // constructor to HTTP client calls openserver + connected = true; + } + + // subclass HttpsClient will overwrite & return an instance of HttpsClient + protected HttpClient getNewHttpClient(URL url, Proxy p, int connectTimeout) + throws IOException { + return HttpClient.New(url, p, connectTimeout, this); + } + + // subclass HttpsClient will overwrite & return an instance of HttpsClient + protected HttpClient getNewHttpClient(URL url, Proxy p, + int connectTimeout, boolean useCache) + throws IOException { + return HttpClient.New(url, p, connectTimeout, useCache, this); + } + + private void expect100Continue() throws IOException { + // Expect: 100-Continue was set, so check the return code for + // Acceptance + int oldTimeout = http.getReadTimeout(); + boolean timedOut = false; + boolean tempTimeOutSet = false; + if (oldTimeout <= 0 || oldTimeout > 5000) { + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine("Timeout currently set to " + + oldTimeout + " temporarily setting it to 5 seconds"); + } + // 5s read timeout in case the server doesn't understand + // Expect: 100-Continue + http.setReadTimeout(5000); + tempTimeOutSet = true; + } + + try { + http.parseHTTP(responses, this); + } catch (SocketTimeoutException se) { + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine("SocketTimeoutException caught," + + " will attempt to send body regardless"); + } + timedOut = true; + } + + if (!timedOut) { + // Can't use getResponseCode() yet + String resp = responses.getValue(0); + // Parse the response which is of the form: + // HTTP/1.1 417 Expectation Failed + // HTTP/1.1 100 Continue + if (resp != null && resp.startsWith("HTTP/")) { + String[] sa = resp.split("\\s+"); + responseCode = -1; + try { + // Response code is 2nd token on the line + if (sa.length > 1) + responseCode = Integer.parseInt(sa[1]); + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine("response code received " + responseCode); + } + if (sa.length > 2) + responseMessage = String.join(" ", Arrays.copyOfRange(sa, 2, sa.length)); + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine("response message received " + responseMessage); + } + } catch (NumberFormatException numberFormatException) { + } + } + if (responseCode != 100) { + // responseCode will be returned to caller + throw new ProtocolException("Server rejected operation"); + } + } + + // If timeout was changed, restore to original value + if (tempTimeOutSet) { + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine("Restoring original timeout : " + oldTimeout); + } + http.setReadTimeout(oldTimeout); + } + + // Ignore any future 100 continue messages + http.setIgnoreContinue(true); + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine("Set Ignore Continue to true"); + } + + responseCode = -1; + responses.reset(); + // Proceed + } + + /* + * Allowable input/output sequences: + * [interpreted as request entity] + * - get output, [write output,] get input, [read input] + * - get output, [write output] + * [interpreted as GET] + * - get input, [read input] + * Disallowed: + * - get input, [read input,] get output, [write output] + */ + + @Override + public OutputStream getOutputStream() throws IOException { + lock(); + try { + connecting = true; + return getOutputStream0(); + } finally { + unlock(); + } + } + + private OutputStream getOutputStream0() throws IOException { + assert isLockHeldByCurrentThread(); + try { + if (!doOutput) { + throw new ProtocolException("cannot write to a URLConnection" + + " if doOutput=false - call setDoOutput(true)"); + } + + if (rememberedExceptionOut != null) { + if (rememberedExceptionOut instanceof RuntimeException) { + throw new RuntimeException(rememberedExceptionOut); + } else { + throw getChainedException((IOException) rememberedExceptionOut); + } + } + + if (method.equals("GET")) { + method = "POST"; // Backward compatibility + } + if ("TRACE".equals(method) && "http".equals(url.getProtocol())) { + throw new ProtocolException("HTTP method TRACE" + + " doesn't support output"); + } + + // if there's already an input stream open, throw an exception + if (inputStream != null) { + throw new ProtocolException("Cannot write output after reading input."); + } + + if (!checkReuseConnection()) + connect(); + + boolean expectContinue = false; + String expects = requests.findValue("Expect"); + if ("100-Continue".equalsIgnoreCase(expects) && streaming()) { + expectContinue = true; + } + + if (streaming() && strOutputStream == null) { + writeRequests(); + } + + if (expectContinue) { + http.setIgnoreContinue(false); + expect100Continue(); + } + ps = (PrintStream)http.getOutputStream(); + if (streaming()) { + if (strOutputStream == null) { + if (chunkLength != -1) { /* chunked */ + strOutputStream = new StreamingOutputStream( + new ChunkedOutputStream(ps, chunkLength), -1L); + } else { /* must be fixed content length */ + long length = 0L; + if (fixedContentLengthLong != -1) { + length = fixedContentLengthLong; + } else if (fixedContentLength != -1) { + length = fixedContentLength; + } + strOutputStream = new StreamingOutputStream(ps, length); + } + } + return strOutputStream; + } else { + if (poster == null) { + poster = new PosterOutputStream(); + } + return poster; + } + } catch (ProtocolException e) { + // Save the response code which may have been set while enforcing + // the 100-continue. disconnectInternal() forces it to -1 + int i = responseCode; + disconnectInternal(); + responseCode = i; + rememberedExceptionOut = e; + throw e; + } catch (RuntimeException | IOException e) { + disconnectInternal(); + rememberedExceptionOut = e; + throw e; + } + } + + // Streaming returns true if there is a request body to send + public boolean streaming () { + return (fixedContentLength != -1) || (fixedContentLengthLong != -1) || + (chunkLength != -1); + } + + /* + * get applicable cookies based on the uri and request headers + * add them to the existing request headers + */ + private void setCookieHeader() throws IOException { + if (cookieHandler != null) { + // we only want to capture the user defined Cookies once, as + // they cannot be changed by user code after we are connected, + // only internally. + + // we should only reach here when called from + // writeRequest, which in turn is only called by + // getInputStream0 + assert isLockHeldByCurrentThread(); + if (setUserCookies) { + int k = requests.getKey("Cookie"); + if (k != -1) + userCookies = requests.getValue(k); + k = requests.getKey("Cookie2"); + if (k != -1) + userCookies2 = requests.getValue(k); + setUserCookies = false; + } + + // remove old Cookie header before setting new one. + requests.remove("Cookie"); + requests.remove("Cookie2"); + + URI uri = ParseUtil.toURI(url); + if (uri != null) { + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + logger.finest("CookieHandler request for " + uri); + } + Map> cookies + = cookieHandler.get( + uri, requests.getHeaders(EXCLUDE_HEADERS)); + if (!cookies.isEmpty()) { + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + logger.finest("Cookies retrieved: " + cookies.toString()); + } + for (Map.Entry> entry : + cookies.entrySet()) { + String key = entry.getKey(); + // ignore all entries that don't have "Cookie" + // or "Cookie2" as keys + if (!"Cookie".equalsIgnoreCase(key) && + !"Cookie2".equalsIgnoreCase(key)) { + continue; + } + List l = entry.getValue(); + if (l != null && !l.isEmpty()) { + requests.add(key, String.join("; ", l)); + } + } + } + } + if (userCookies != null) { + int k; + if ((k = requests.getKey("Cookie")) != -1) + requests.set("Cookie", requests.getValue(k) + ";" + userCookies); + else + requests.set("Cookie", userCookies); + } + if (userCookies2 != null) { + int k; + if ((k = requests.getKey("Cookie2")) != -1) + requests.set("Cookie2", requests.getValue(k) + ";" + userCookies2); + else + requests.set("Cookie2", userCookies2); + } + + } // end of getting cookies + } + + @Override + public InputStream getInputStream() throws IOException { + lock(); + try { + connecting = true; + return getInputStream0(); + } finally { + unlock(); + } + } + + @SuppressWarnings("empty-statement") + private InputStream getInputStream0() throws IOException { + + assert isLockHeldByCurrentThread(); + if (!doInput) { + throw new ProtocolException("Cannot read from URLConnection" + + " if doInput=false (call setDoInput(true))"); + } + + if (rememberedException != null) { + if (rememberedException instanceof RuntimeException) + throw new RuntimeException(rememberedException); + else { + throw getChainedException((IOException)rememberedException); + } + } + + if (inputStream != null) { + return inputStream; + } + + if (streaming() ) { + if (strOutputStream == null) { + getOutputStream(); + } + /* make sure stream is closed */ + strOutputStream.close (); + if (!strOutputStream.writtenOK()) { + throw new IOException ("Incomplete output stream"); + } + } + + int redirects = 0; + int respCode = 0; + long cl = -1; + AuthenticationInfo serverAuthentication = null; + AuthenticationInfo proxyAuthentication = null; + AuthenticationHeader srvHdr = null; + + /** + * Failed Negotiate + * + * In some cases, the Negotiate auth is supported for the + * remote host but the negotiate process still fails (For + * example, if the web page is located on a backend server + * and delegation is needed but fails). The authentication + * process will start again, and we need to detect this + * kind of failure and do proper fallback (say, to NTLM). + * + * In order to achieve this, the inNegotiate flag is set + * when the first negotiate challenge is met (and reset + * if authentication is finished). If a fresh new negotiate + * challenge (no parameter) is found while inNegotiate is + * set, we know there's a failed auth attempt recently. + * Here we'll ignore the header line so that fallback + * can be practiced. + * + * inNegotiateProxy is for proxy authentication. + */ + boolean inNegotiate = false; + boolean inNegotiateProxy = false; + + // If the user has set either of these headers then do not remove them + isUserServerAuth = requests.getKey("Authorization") != -1; + isUserProxyAuth = requests.getKey("Proxy-Authorization") != -1; + try { + do { + if (!checkReuseConnection()) + connect(); + + if (cachedInputStream != null) { + return cachedInputStream; + } + + // we may need to remove proxy-authorization + Proxy p = http.getHttpProxy(); + // if we're not using a proxy or if the proxy to be used is not + // the same as the originally set one, then remove it + if (p == null || (lastProxy != null && !lastProxy.equals(p))) { + requests.remove("Proxy-Authorization"); + lastProxy = null; + } + /* REMIND: This exists to fix the HttpsURLConnection subclass. + * Hotjava needs to run on JDK1.1FCS. Do proper fix once a + * proper solution for SSL can be found. + */ + ps = (PrintStream)http.getOutputStream(); + + if (!streaming()) { + writeRequests(); + } + http.parseHTTP(responses, this); + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine(responses.toString()); + } + + boolean b1 = responses.filterNTLMResponses("WWW-Authenticate"); + boolean b2 = responses.filterNTLMResponses("Proxy-Authenticate"); + if (b1 || b2) { + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine(">>>> Headers are filtered"); + logger.fine(responses.toString()); + } + } + + inputStream = http.getInputStream(); + + respCode = getResponseCode(); + if (respCode == -1) { + disconnectInternal(); + throw new IOException ("Invalid Http response"); + } + if (respCode == HTTP_PROXY_AUTH && tunnelState() != TunnelState.TUNNELING) { + if (streaming()) { + disconnectInternal(); + throw new HttpRetryException ( + RETRY_MSG1, HTTP_PROXY_AUTH); + } + + // Read comments labeled "Failed Negotiate" for details. + boolean dontUseNegotiate = false; + Iterator iter = responses.multiValueIterator("Proxy-Authenticate"); + while (iter.hasNext()) { + String value = iter.next().trim(); + if (value.equalsIgnoreCase("Negotiate") || + value.equalsIgnoreCase("Kerberos")) { + if (!inNegotiateProxy) { + inNegotiateProxy = true; + } else { + dontUseNegotiate = true; + doingNTLMp2ndStage = false; + proxyAuthentication = null; + } + break; + } + } + + // changes: add a 3rd parameter to the constructor of + // AuthenticationHeader, so that NegotiateAuthentication. + // isSupported can be tested. + // The other 2 appearances of "new AuthenticationHeader" is + // altered in similar ways. + + AuthenticationHeader authhdr = new AuthenticationHeader ( + "Proxy-Authenticate", + responses, + getHttpCallerInfo(url, + http.getProxyHostUsed(), + http.getProxyPortUsed(), + authenticator), + dontUseNegotiate, + disabledProxyingSchemes + ); + + if (!doingNTLMp2ndStage) { + proxyAuthentication = + resetProxyAuthentication(proxyAuthentication, authhdr); + if (proxyAuthentication != null) { + redirects++; + disconnectInternal(); + continue; + } + } else { + /* in this case, only one header field will be present */ + String raw = responses.findValue ("Proxy-Authenticate"); + reset (); + try { + proxyAuthentication.setHeaders(this, + authhdr.headerParser(), raw); + } catch (IOException ex) { + disconnectInternal(); + if (Exceptions.enhancedNonSocketExceptions()) { + throw new IOException ("Authentication failure", ex); + } else { + throw new IOException ("Authentication failure"); + } + } + if (serverAuthentication != null && srvHdr != null) { + try { + serverAuthentication.setHeaders(this, + srvHdr.headerParser(), raw); + } catch (IOException ex) { + disconnectInternal(); + if (Exceptions.enhancedNonSocketExceptions()) { + throw new IOException ("Authentication failure", ex); + } else { + throw new IOException ("Authentication failure"); + } + } + } + authObj = null; + doingNTLMp2ndStage = false; + continue; + } + } else { + inNegotiateProxy = false; + doingNTLMp2ndStage = false; + if (!isUserProxyAuth) + requests.remove("Proxy-Authorization"); + } + + // cache proxy authentication info + if (proxyAuthentication != null) { + // cache auth info on success, domain header not relevant. + proxyAuthentication.addToCache(authCache); + } + + if (respCode == HTTP_UNAUTHORIZED) { + if (streaming()) { + disconnectInternal(); + throw new HttpRetryException ( + RETRY_MSG2, HTTP_UNAUTHORIZED); + } + + // Read comments labeled "Failed Negotiate" for details. + boolean dontUseNegotiate = false; + Iterator iter = responses.multiValueIterator("WWW-Authenticate"); + while (iter.hasNext()) { + String value = iter.next().trim(); + if (value.equalsIgnoreCase("Negotiate") || + value.equalsIgnoreCase("Kerberos")) { + if (!inNegotiate) { + inNegotiate = true; + } else { + dontUseNegotiate = true; + doingNTLM2ndStage = false; + serverAuthentication = null; + } + break; + } + } + + srvHdr = new AuthenticationHeader ( + "WWW-Authenticate", responses, + getHttpCallerInfo(url, authenticator), + dontUseNegotiate + ); + + String raw = srvHdr.raw(); + if (!doingNTLM2ndStage) { + if ((serverAuthentication != null)&& + serverAuthentication.getAuthScheme() != NTLM) { + if (serverAuthentication.isAuthorizationStale (raw)) { + /* we can retry with the current credentials */ + disconnectWeb(); + redirects++; + requests.set(serverAuthentication.getHeaderName(), + serverAuthentication.getHeaderValue(url, method)); + currentServerCredentials = serverAuthentication; + setCookieHeader(); + continue; + } else { + serverAuthentication.removeFromCache(authCache); + } + } + serverAuthentication = getServerAuthentication(srvHdr); + currentServerCredentials = serverAuthentication; + + if (serverAuthentication != null) { + disconnectWeb(); + redirects++; // don't let things loop ad nauseum + setCookieHeader(); + continue; + } + } else { + reset (); + /* header not used for ntlm */ + try { + serverAuthentication.setHeaders(this, null, raw); + } catch (IOException ex) { + disconnectWeb(); + if (Exceptions.enhancedNonSocketExceptions()) { + throw new IOException ("Authentication failure", ex); + } else { + throw new IOException ("Authentication failure"); + } + } + doingNTLM2ndStage = false; + authObj = null; + setCookieHeader(); + continue; + } + } + // cache server authentication info + if (serverAuthentication != null) { + // cache auth info on success + if (!(serverAuthentication instanceof DigestAuthentication) || + (domain == null)) { + if (serverAuthentication instanceof BasicAuthentication) { + // check if the path is shorter than the existing entry + String npath = AuthenticationInfo.reducePath (url.getPath()); + String opath = serverAuthentication.path; + if (!opath.startsWith (npath) || npath.length() >= opath.length()) { + /* npath is longer, there must be a common root */ + npath = BasicAuthentication.getRootPath (opath, npath); + } + // remove the entry and create a new one + BasicAuthentication a = + (BasicAuthentication) serverAuthentication.clone(); + serverAuthentication.removeFromCache(authCache); + a.path = npath; + serverAuthentication = a; + } + serverAuthentication.addToCache(authCache); + } else { + // what we cache is based on the domain list in the request + DigestAuthentication srv = (DigestAuthentication) + serverAuthentication; + StringTokenizer tok = new StringTokenizer (domain," "); + String realm = srv.realm; + PasswordAuthentication pw = srv.pw; + digestparams = srv.params; + while (tok.hasMoreTokens()) { + String path = tok.nextToken(); + try { + /* path could be an abs_path or a complete URI */ + URL u = newURL (url, path); + DigestAuthentication d = new DigestAuthentication ( + false, u, realm, "Digest", pw, + digestparams); + d.addToCache (authCache); + } catch (Exception e) {} + } + } + } + + // some flags should be reset to its initialized form so that + // even after a redirect the necessary checks can still be + // performed. + inNegotiate = false; + inNegotiateProxy = false; + + //serverAuthentication = null; + doingNTLMp2ndStage = false; + doingNTLM2ndStage = false; + if (!isUserServerAuth) + requests.remove("Authorization"); + if (!isUserProxyAuth) + requests.remove("Proxy-Authorization"); + + if (respCode == HTTP_OK) { + checkResponseCredentials (false); + } else { + needToCheck = false; + } + + // a flag need to clean + needToCheck = true; + + if (followRedirect()) { + /* if we should follow a redirect, then the followRedirects() + * method will disconnect() and re-connect us to the new + * location + */ + redirects++; + + // redirecting HTTP response may have set cookie, so + // need to re-generate request header + setCookieHeader(); + + continue; + } + + final String contentLengthVal = responses.findValue("content-length"); + if (contentLengthVal != null) { + try { + cl = Long.parseLong(contentLengthVal); + } catch (NumberFormatException nfe) { } + } + + if (method.equals("HEAD") || cl == 0 || + respCode == HTTP_NOT_MODIFIED || + respCode == HTTP_NO_CONTENT) { + noResponseBody(); + } + + if (respCode == 200 || respCode == 203 || respCode == 206 || + respCode == 300 || respCode == 301 || respCode == 410) { + if (cacheHandler != null && getUseCaches()) { + // give cache a chance to save response in cache + URI uri = ParseUtil.toURI(url); + if (uri != null) { + URLConnection uconn = this; + if ("https".equalsIgnoreCase(uri.getScheme())) { + try { + // use reflection to get to the public + // HttpsURLConnection instance saved in + // DelegateHttpsURLConnection + uconn = (URLConnection)this.getClass().getField("httpsURLConnection").get(this); + } catch (IllegalAccessException | + NoSuchFieldException e) { + // ignored; use 'this' + } + } + CacheRequest cacheRequest = + cacheHandler.put(uri, uconn); + if (cacheRequest != null && http != null) { + http.setCacheRequest(cacheRequest); + inputStream = new HttpInputStream(inputStream, cacheRequest); + } + } + } + } + + if (!(inputStream instanceof HttpInputStream)) { + inputStream = new HttpInputStream(inputStream); + } + + if (respCode >= 400) { + if (respCode == 404 || respCode == 410) { + throw new FileNotFoundException(url.toString()); + } else { + throw new java.io.IOException("Server returned HTTP" + + " response code: " + respCode + " for URL: " + + url.toString()); + } + } + poster = null; + strOutputStream = null; + return inputStream; + } while (redirects < maxRedirects); + + throw new ProtocolException("Server redirected too many times (" + + redirects + ")"); + } catch (RuntimeException e) { + disconnectInternal(); + rememberedException = e; + throw e; + } catch (IOException e) { + rememberedException = e; + + // buffer the error stream if bytes < 4k + // and it can be buffered within 1 second + String te = responses.findValue("Transfer-Encoding"); + if (http != null && http.isKeepingAlive() && enableESBuffer && + (cl > 0 || (te != null && te.equalsIgnoreCase("chunked")))) { + errorStream = ErrorStream.getErrorStream(inputStream, cl, http); + } + throw e; + } finally { + if (proxyAuthKey != null) { + AuthenticationInfo.endAuthRequest(proxyAuthKey); + } + if (serverAuthKey != null) { + AuthenticationInfo.endAuthRequest(serverAuthKey); + } + if (proxyAuthentication != null) { + proxyAuthentication.disposeContext(); + } + if (serverAuthentication != null) { + serverAuthentication.disposeContext(); + } + } + } + + /** + * This method is called when a response with no response + * body is received, and arrange for the http client to + * be returned to the pool (or released) immediately when + * possible. + * @apiNote Used by {@link sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection} + * to preserve the TLS information after receiving an empty body. + * @implSpec + * Subclasses that override this method should call the super class + * implementation. + */ + protected void noResponseBody() { + http.finished(); + http = null; + inputStream = new EmptyInputStream(); + connected = false; + } + + /* + * Creates a chained exception that has the same type as + * original exception and with the same message. Right now, + * there is no convenient APIs for doing so. + */ + private IOException getChainedException(final IOException rememberedException) { + try { + final Object[] args = { rememberedException.getMessage() }; + IOException chainedException = rememberedException.getClass() + .getConstructor(new Class[] { String.class }) + .newInstance(args); + chainedException.initCause(rememberedException); + return chainedException; + } catch (Exception ignored) { + return rememberedException; + } + } + + @Override + public InputStream getErrorStream() { + if (connected && responseCode >= 400) { + // Client Error 4xx and Server Error 5xx + if (errorStream != null) { + return errorStream; + } else if (inputStream != null) { + return inputStream; + } + } + return null; + } + + /** + * set or reset proxy authentication info in request headers + * after receiving a 407 error. In the case of NTLM however, + * receiving a 407 is normal and we just skip the stale check + * because ntlm does not support this feature. + */ + private AuthenticationInfo + resetProxyAuthentication(AuthenticationInfo proxyAuthentication, AuthenticationHeader auth) throws IOException { + + // Only called from getInputStream0 and doTunneling0 + assert isLockHeldByCurrentThread(); + + if ((proxyAuthentication != null )&& + proxyAuthentication.getAuthScheme() != NTLM) { + String raw = auth.raw(); + if (proxyAuthentication.isAuthorizationStale (raw)) { + /* we can retry with the current credentials */ + String value; + if (proxyAuthentication instanceof DigestAuthentication) { + DigestAuthentication digestProxy = (DigestAuthentication) + proxyAuthentication; + if (tunnelState() == TunnelState.SETUP) { + value = digestProxy.getHeaderValue(connectRequestURI(url), HTTP_CONNECT); + } else { + value = digestProxy.getHeaderValue(getRequestURI(), method); + } + } else { + value = proxyAuthentication.getHeaderValue(url, method); + } + requests.set(proxyAuthentication.getHeaderName(), value); + currentProxyCredentials = proxyAuthentication; + return proxyAuthentication; + } else { + proxyAuthentication.removeFromCache(authCache); + } + } + proxyAuthentication = getHttpProxyAuthentication(auth); + currentProxyCredentials = proxyAuthentication; + return proxyAuthentication; + } + + /** + * Returns the tunnel state. + * + * @return the state + */ + TunnelState tunnelState() { + return tunnelState; + } + + /** + * Set the tunneling status. + * + * @param tunnelState the state + */ + public void setTunnelState(TunnelState tunnelState) { + this.tunnelState = tunnelState; + } + + /** + * establish a tunnel through proxy server + */ + public void doTunneling() throws IOException { + lock(); + try { + doTunneling0(); + } finally{ + unlock(); + } + } + + private void doTunneling0() throws IOException { + int retryTunnel = 0; + String statusLine = ""; + int respCode = 0; + AuthenticationInfo proxyAuthentication = null; + String proxyHost = null; + int proxyPort = -1; + + assert isLockHeldByCurrentThread(); + + // save current requests so that they can be restored after tunnel is setup. + MessageHeader savedRequests = requests; + requests = new MessageHeader(); + + // Read comments labeled "Failed Negotiate" for details. + boolean inNegotiateProxy = false; + + try { + /* Actively setting up a tunnel */ + setTunnelState(TunnelState.SETUP); + + do { + if (!checkReuseConnection()) { + proxiedConnect(url, proxyHost, proxyPort, false); + } + // send the "CONNECT" request to establish a tunnel + // through proxy server + sendCONNECTRequest(); + responses.reset(); + + http.parseHTTP(responses, this); + + /* Log the response to the CONNECT */ + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine(responses.toString()); + } + + if (responses.filterNTLMResponses("Proxy-Authenticate")) { + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine(">>>> Headers are filtered"); + logger.fine(responses.toString()); + } + } + + statusLine = responses.getValue(0); + respCode = parseConnectResponseCode(statusLine); + if (respCode == -1) { + // a respCode of -1, due to a invalid status line, + // will (rightly) result in an IOException being thrown + // later in this code. here we merely log the invalid status line. + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine("invalid status line: \"" + statusLine + "\""); + } + } + if (respCode == HTTP_PROXY_AUTH) { + // Read comments labeled "Failed Negotiate" for details. + boolean dontUseNegotiate = false; + Iterator iter = responses.multiValueIterator("Proxy-Authenticate"); + while (iter.hasNext()) { + String value = iter.next().trim(); + if (value.equalsIgnoreCase("Negotiate") || + value.equalsIgnoreCase("Kerberos")) { + if (!inNegotiateProxy) { + inNegotiateProxy = true; + } else { + dontUseNegotiate = true; + doingNTLMp2ndStage = false; + proxyAuthentication = null; + } + break; + } + } + + AuthenticationHeader authhdr = new AuthenticationHeader( + "Proxy-Authenticate", + responses, + getHttpCallerInfo(url, + http.getProxyHostUsed(), + http.getProxyPortUsed(), + authenticator), + dontUseNegotiate, + disabledTunnelingSchemes + ); + if (!doingNTLMp2ndStage) { + proxyAuthentication = + resetProxyAuthentication(proxyAuthentication, authhdr); + if (proxyAuthentication != null) { + proxyHost = http.getProxyHostUsed(); + proxyPort = http.getProxyPortUsed(); + disconnectInternal(); + retryTunnel++; + continue; + } + } else { + String raw = responses.findValue ("Proxy-Authenticate"); + reset (); + try { + proxyAuthentication.setHeaders(this, + authhdr.headerParser(), raw); + } catch (IOException ex) { + disconnectInternal(); + if (Exceptions.enhancedNonSocketExceptions()) { + throw new IOException ("Authentication failure", ex); + } else { + throw new IOException ("Authentication failure"); + } + } + authObj = null; + doingNTLMp2ndStage = false; + continue; + } + } + // cache proxy authentication info + if (proxyAuthentication != null) { + // cache auth info on success, domain header not relevant. + proxyAuthentication.addToCache(authCache); + } + + if (respCode == HTTP_OK) { + setTunnelState(TunnelState.TUNNELING); + savedRequests.remove("Proxy-Authorization"); + break; + } + // we don't know how to deal with other response code + // so disconnect and report error + disconnectInternal(); + setTunnelState(TunnelState.NONE); + break; + } while (retryTunnel < maxRedirects); + + if (retryTunnel >= maxRedirects || (respCode != HTTP_OK)) { + if (respCode != HTTP_PROXY_AUTH) { + // remove all but authenticate responses + responses.reset(); + } + throw new IOException("Unable to tunnel through proxy."+ + " Proxy returns \"" + + statusLine + "\""); + } + } finally { + if (proxyAuthKey != null) { + AuthenticationInfo.endAuthRequest(proxyAuthKey); + } + if (proxyAuthentication != null) { + proxyAuthentication.disposeContext(); + } + } + + // restore original request headers + requests = savedRequests; + + // reset responses + responses.reset(); + } + + // parses the status line, that was returned for a CONNECT request, and returns + // the response code from that line. returns -1 if the response code could not be + // parsed. + private static int parseConnectResponseCode(final String statusLine) { + final int invalidStatusLine = -1; + if (statusLine == null || statusLine.isBlank()) { + return invalidStatusLine; + } + // + // status-line = HTTP-version SP status-code SP [ reason-phrase ] + // SP = space character + // + final StringTokenizer st = new StringTokenizer(statusLine, " "); + if (!st.hasMoreTokens()) { + return invalidStatusLine; + } + st.nextToken(); // the HTTP version part (ex: HTTP/1.1) + if (!st.hasMoreTokens()) { + return invalidStatusLine; + } + final String v = st.nextToken().trim(); // status code + try { + return Integer.parseInt(v); + } catch (NumberFormatException nfe) { + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine("invalid response code: " + v); + } + } + return invalidStatusLine; + } + + /** + * Overridden in https to also include the server certificate + */ + protected HttpCallerInfo getHttpCallerInfo(URL url, String proxy, int port, + Authenticator authenticator) { + return new HttpCallerInfo(url, proxy, port, authenticator); + } + + /** + * Overridden in https to also include the server certificate + */ + protected HttpCallerInfo getHttpCallerInfo(URL url, Authenticator authenticator) { + return new HttpCallerInfo(url, authenticator); + } + + static String connectRequestURI(URL url) { + String host = url.getHost(); + int port = url.getPort(); + port = port != -1 ? port : url.getDefaultPort(); + + return host + ":" + port; + } + + /** + * send a CONNECT request for establishing a tunnel to proxy server + */ + private void sendCONNECTRequest() throws IOException { + int port = url.getPort(); + + requests.set(0, HTTP_CONNECT + " " + connectRequestURI(url) + + " " + httpVersion, null); + requests.setIfNotSet("User-Agent", userAgent); + + String host = url.getHost(); + if (port != -1 && port != url.getDefaultPort()) { + host += ":" + String.valueOf(port); + } + requests.setIfNotSet("Host", host); + + // Not really necessary for a tunnel, but can't hurt + requests.setIfNotSet("Accept", acceptString); + + if (http.getHttpKeepAliveSet()) { + requests.setIfNotSet("Proxy-Connection", "keep-alive"); + } + + setPreemptiveProxyAuthentication(requests); + + /* Log the CONNECT request */ + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine(requests.toString()); + } + + http.writeRequests(requests, null); + } + + /** + * Sets pre-emptive proxy authentication in header + */ + private void setPreemptiveProxyAuthentication(MessageHeader requests) throws IOException { + AuthenticationInfo pauth + = AuthenticationInfo.getProxyAuth(http.getProxyHostUsed(), + http.getProxyPortUsed(), + authCache); + if (pauth != null && pauth.supportsPreemptiveAuthorization()) { + String value; + if (pauth instanceof DigestAuthentication) { + DigestAuthentication digestProxy = (DigestAuthentication) pauth; + if (tunnelState() == TunnelState.SETUP) { + value = digestProxy + .getHeaderValue(connectRequestURI(url), HTTP_CONNECT); + } else { + value = digestProxy.getHeaderValue(getRequestURI(), method); + } + } else { + value = pauth.getHeaderValue(url, method); + } + + // Sets "Proxy-authorization" + requests.set(pauth.getHeaderName(), value); + currentProxyCredentials = pauth; + } + } + + /** + * Gets the authentication for an HTTP proxy, and applies it to + * the connection. + */ + private AuthenticationInfo getHttpProxyAuthentication(AuthenticationHeader authhdr) + throws IOException { + + assert isLockHeldByCurrentThread(); + + /* get authorization from authenticator */ + AuthenticationInfo ret = null; + String raw = authhdr.raw(); + String host = http.getProxyHostUsed(); + int port = http.getProxyPortUsed(); + if (host != null && authhdr.isPresent()) { + HeaderParser p = authhdr.headerParser(); + String realm = p.findValue("realm"); + String charset = p.findValue("charset"); + boolean isUTF8 = (charset != null && charset.equalsIgnoreCase("UTF-8")); + String scheme = authhdr.scheme(); + AuthScheme authScheme = UNKNOWN; + if ("basic".equalsIgnoreCase(scheme)) { + authScheme = BASIC; + } else if ("digest".equalsIgnoreCase(scheme)) { + authScheme = DIGEST; + } else if ("ntlm".equalsIgnoreCase(scheme)) { + authScheme = NTLM; + doingNTLMp2ndStage = true; + } else if ("Kerberos".equalsIgnoreCase(scheme)) { + authScheme = KERBEROS; + doingNTLMp2ndStage = true; + } else if ("Negotiate".equalsIgnoreCase(scheme)) { + authScheme = NEGOTIATE; + doingNTLMp2ndStage = true; + } + + if (realm == null) + realm = ""; + proxyAuthKey = AuthenticationInfo.getProxyAuthKey(host, port, realm, authScheme); + ret = AuthenticationInfo.getProxyAuth(proxyAuthKey, authCache); + if (ret == null) { + ret = switch (authScheme) { + case BASIC -> { + InetAddress addr = null; + try { + addr = InetAddress.getByName(host); + } catch (UnknownHostException uhe) { + // Ignore the exception. The Authenticator instance will + // be passed a null InetAddress when requesting a password from the + // Authenticator. + } + final PasswordAuthentication a = requestPassword(authenticator, + host, addr, port, "http", + realm, scheme, url, RequestorType.PROXY); + if (a != null) { + yield new BasicAuthentication(true, host, port, realm, a, isUTF8); + } + yield null; + } + case DIGEST -> { + final PasswordAuthentication a = requestPassword(authenticator, + host, null, port, url.getProtocol(), + realm, scheme, url, RequestorType.PROXY); + if (a != null) { + DigestAuthentication.Parameters params = + new DigestAuthentication.Parameters(); + yield new DigestAuthentication(true, host, port, realm, + scheme, a, params); + } + yield null; + } + case NTLM -> { + if (!NTLMAuthenticationProxy.supported) { + yield null; + } + /* tryTransparentNTLMProxy will always be true the first + * time around, but verify that the platform supports it + * otherwise don't try. */ + if (tryTransparentNTLMProxy) { + tryTransparentNTLMProxy = + NTLMAuthenticationProxy.supportsTransparentAuth; + /* If the platform supports transparent authentication + * then normally it's ok to do transparent auth to a proxy + * because we generally trust proxies (chosen by the user) + * But not in the case of 305 response where the server + * chose it. */ + if (tryTransparentNTLMProxy && useProxyResponseCode) { + tryTransparentNTLMProxy = false; + } + + } + PasswordAuthentication a = null; + if (tryTransparentNTLMProxy) { + logger.finest("Trying Transparent NTLM authentication"); + } else { + a = requestPassword( + authenticator, + host, null, port, url.getProtocol(), + "", scheme, url, RequestorType.PROXY); + validateNTLMCredentials(a); + } + /* If we are not trying transparent authentication then + * we need to have a PasswordAuthentication instance. For + * transparent authentication (Windows only) the username + * and password will be picked up from the current logged + * on users credentials. + */ + AuthenticationInfo authInfo = null; + if (tryTransparentNTLMProxy || + (!tryTransparentNTLMProxy && a != null)) { + authInfo = NTLMAuthenticationProxy.proxy.create(true, host, port, a); + } + + /* set to false so that we do not try again */ + tryTransparentNTLMProxy = false; + yield authInfo; + } + case NEGOTIATE -> + new NegotiateAuthentication(new HttpCallerInfo(authhdr.getHttpCallerInfo(), "Negotiate")); + case KERBEROS -> + new NegotiateAuthentication(new HttpCallerInfo(authhdr.getHttpCallerInfo(), "Kerberos")); + case UNKNOWN -> { + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + logger.finest("Unknown/Unsupported authentication scheme: " + scheme); + } + yield null; + } + }; + } + if (ret != null) { + try { + ret.setHeaders(this, p, raw); + } catch (IOException e) { + ret.disposeContext(); + ret = null; + } + } + } + if (logger.isLoggable(PlatformLogger.Level.FINER)) { + logger.finer("Proxy Authentication for " + authhdr.toString() +" returned " + (ret != null ? ret.toString() : "null")); + } + return ret; + } + + /** + * Gets the authentication for an HTTP server, and applies it to + * the connection. + * @param authhdr the AuthenticationHeader which tells what auth scheme is + * preferred. + */ + @SuppressWarnings("fallthrough") + private AuthenticationInfo getServerAuthentication(AuthenticationHeader authhdr) + throws IOException { + + // Only called from getInputStream0 + assert isLockHeldByCurrentThread(); + + /* get authorization from authenticator */ + AuthenticationInfo ret = null; + String raw = authhdr.raw(); + /* When we get an NTLM auth from cache, don't set any special headers */ + if (authhdr.isPresent()) { + HeaderParser p = authhdr.headerParser(); + String realm = p.findValue("realm"); + String scheme = authhdr.scheme(); + String charset = p.findValue("charset"); + boolean isUTF8 = (charset != null && charset.equalsIgnoreCase("UTF-8")); + AuthScheme authScheme = UNKNOWN; + if ("basic".equalsIgnoreCase(scheme)) { + authScheme = BASIC; + } else if ("digest".equalsIgnoreCase(scheme)) { + authScheme = DIGEST; + } else if ("ntlm".equalsIgnoreCase(scheme)) { + authScheme = NTLM; + doingNTLM2ndStage = true; + } else if ("Kerberos".equalsIgnoreCase(scheme)) { + authScheme = KERBEROS; + doingNTLM2ndStage = true; + } else if ("Negotiate".equalsIgnoreCase(scheme)) { + authScheme = NEGOTIATE; + doingNTLM2ndStage = true; + } + + domain = p.findValue ("domain"); + if (realm == null) + realm = ""; + serverAuthKey = AuthenticationInfo.getServerAuthKey(url, realm, authScheme); + ret = AuthenticationInfo.getServerAuth(serverAuthKey, authCache); + InetAddress addr = null; + if (ret == null) { + try { + addr = InetAddress.getByName(url.getHost()); + } catch (java.net.UnknownHostException ignored) { + // User will have addr = null + } + } + // replacing -1 with default port for a protocol + int port = url.getPort(); + if (port == -1) { + port = url.getDefaultPort(); + } + if (ret == null) { + switch(authScheme) { + case KERBEROS: + ret = new NegotiateAuthentication(new HttpCallerInfo(authhdr.getHttpCallerInfo(), "Kerberos")); + break; + case NEGOTIATE: + ret = new NegotiateAuthentication(new HttpCallerInfo(authhdr.getHttpCallerInfo(), "Negotiate")); + break; + case BASIC: + PasswordAuthentication a = + requestPassword( + authenticator, + url.getHost(), addr, port, url.getProtocol(), + realm, scheme, url, RequestorType.SERVER); + if (a != null) { + ret = new BasicAuthentication(false, url, realm, a, isUTF8); + } + break; + case DIGEST: + a = requestPassword( + authenticator, + url.getHost(), addr, port, url.getProtocol(), + realm, scheme, url, RequestorType.SERVER); + if (a != null) { + digestparams = new DigestAuthentication.Parameters(); + ret = new DigestAuthentication(false, url, realm, scheme, + a, digestparams); + } + break; + case NTLM: + if (NTLMAuthenticationProxy.supported) { + URL url1; + try { + url1 = newURL (url, "/"); /* truncate the path */ + } catch (Exception e) { + url1 = url; + } + + /* tryTransparentNTLMServer will always be true the first + * time around, but verify that the platform supports it + * otherwise don't try. */ + if (tryTransparentNTLMServer) { + tryTransparentNTLMServer = + NTLMAuthenticationProxy.supportsTransparentAuth; + /* If the platform supports transparent authentication + * then check if we are in a secure environment + * whether, or not, we should try transparent authentication.*/ + if (tryTransparentNTLMServer) { + tryTransparentNTLMServer = + NTLMAuthenticationProxy.isTrustedSite(url); + } + } + a = null; + if (tryTransparentNTLMServer) { + logger.finest("Trying Transparent NTLM authentication"); + } else { + a = requestPassword( + authenticator, + url.getHost(), addr, port, url.getProtocol(), + "", scheme, url, RequestorType.SERVER); + validateNTLMCredentials(a); + } + + /* If we are not trying transparent authentication then + * we need to have a PasswordAuthentication instance. For + * transparent authentication (Windows only) the username + * and password will be picked up from the current logged + * on users credentials. + */ + if (tryTransparentNTLMServer || + (!tryTransparentNTLMServer && a != null)) { + ret = NTLMAuthenticationProxy.proxy.create(false, url1, a); + } + + /* set to false so that we do not try again */ + tryTransparentNTLMServer = false; + } + break; + case UNKNOWN: + if (logger.isLoggable(PlatformLogger.Level.FINEST)) { + logger.finest("Unknown/Unsupported authentication scheme: " + scheme); + } + /*fall through*/ + default: + throw new AssertionError("should not reach here"); + } + } + if (ret != null ) { + try { + ret.setHeaders(this, p, raw); + } catch (IOException e) { + ret.disposeContext(); + ret = null; + } + } + } + if (logger.isLoggable(PlatformLogger.Level.FINER)) { + logger.finer("Server Authentication for " + authhdr.toString() +" returned " + (ret != null ? ret.toString() : "null")); + } + return ret; + } + + /* inclose will be true if called from close(), in which case we + * force the call to check because this is the last chance to do so. + * If not in close(), then the authentication info could arrive in a trailer + * field, which we have not read yet. + */ + private void checkResponseCredentials (boolean inClose) throws IOException { + try { + if (!needToCheck) + return; + if ((validateProxy && currentProxyCredentials != null) && + (currentProxyCredentials instanceof DigestAuthentication)) { + String raw = responses.findValue ("Proxy-Authentication-Info"); + if (inClose || (raw != null)) { + DigestAuthentication da = (DigestAuthentication) + currentProxyCredentials; + da.checkResponse (raw, method, getRequestURI()); + currentProxyCredentials.disposeContext(); + currentProxyCredentials = null; + } + } + if ((validateServer && currentServerCredentials != null) && + (currentServerCredentials instanceof DigestAuthentication)) { + String raw = responses.findValue ("Authentication-Info"); + if (inClose || (raw != null)) { + DigestAuthentication da = (DigestAuthentication) + currentServerCredentials; + da.checkResponse (raw, method, url); + currentServerCredentials.disposeContext(); + currentServerCredentials = null; + } + } + if ((currentServerCredentials==null) && (currentProxyCredentials == null)) { + needToCheck = false; + } + } catch (IOException e) { + disconnectInternal(); + connected = false; + throw e; + } + } + + /* The request URI used in the request line for this request. + * Also, needed for digest authentication + */ + + String requestURI = null; + + String getRequestURI() throws IOException { + if (requestURI == null) { + requestURI = http.getURLFile(); + } + return requestURI; + } + + /* Tells us whether to follow a redirect. If so, it + * closes the connection (break any keep-alive) and + * resets the url, re-connects, and resets the request + * property. + */ + private boolean followRedirect() throws IOException { + if (!getInstanceFollowRedirects()) { + return false; + } + + final int stat = getResponseCode(); + if (stat < 300 || stat > 307 || stat == 306 + || stat == HTTP_NOT_MODIFIED) { + return false; + } + final String loc = getHeaderField("Location"); + if (loc == null) { + /* this should be present - if not, we have no choice + * but to go forward w/ the response we got + */ + return false; + } + + URL locUrl; + try { + locUrl = newURL(loc); + if (!url.getProtocol().equalsIgnoreCase(locUrl.getProtocol())) { + return false; + } + + } catch (MalformedURLException mue) { + // treat loc as a relative URI to conform to popular browsers + locUrl = newURL(url, loc); + } + return followRedirect0(loc, stat, locUrl); + } + + /* Tells us whether to follow a redirect. If so, it + * closes the connection (break any keep-alive) and + * resets the url, re-connects, and resets the request + * property. + */ + private boolean followRedirect0(String loc, int stat, URL locUrl) + throws IOException + { + assert isLockHeldByCurrentThread(); + + lastProxy = http.getHttpProxy(); + disconnectInternal(); + if (streaming()) { + throw new HttpRetryException (RETRY_MSG3, stat, loc); + } + if (logger.isLoggable(PlatformLogger.Level.FINE)) { + logger.fine("Redirected from " + url + " to " + locUrl); + } + + // clear out old response headers!!!! + responses = new MessageHeader(maxHeaderSize); + if (stat == HTTP_USE_PROXY) { + /* This means we must re-request the resource through the + * proxy denoted in the "Location:" field of the response. + * Judging by the spec, the string in the Location header + * _should_ denote a URL - let's hope for "http://my.proxy.org" + * Make a new HttpClient to the proxy, using HttpClient's + * Instance-specific proxy fields, but note we're still fetching + * the same URL. + */ + String proxyHost = locUrl.getHost(); + int proxyPort = locUrl.getPort(); + + setProxiedClient (url, proxyHost, proxyPort); + requests.set(0, method + " " + getRequestURI()+" " + + httpVersion, null); + connected = true; + // need to remember this in case NTLM proxy authentication gets + // used. We can't use transparent authentication when user + // doesn't know about proxy. + useProxyResponseCode = true; + } else { + final URL prevURL = url; + + // maintain previous headers, just change the name + // of the file we're getting + url = locUrl; + requestURI = null; // force it to be recalculated + if (method.equals("POST") && !Boolean.getBoolean("http.strictPostRedirect") && (stat!=307)) { + /* The HTTP/1.1 spec says that a redirect from a POST + * *should not* be immediately turned into a GET, and + * that some HTTP/1.0 clients incorrectly did this. + * Correct behavior redirects a POST to another POST. + * Unfortunately, since most browsers have this incorrect + * behavior, the web works this way now. Typical usage + * seems to be: + * POST a login code or passwd to a web page. + * after validation, the server redirects to another + * (welcome) page + * The second request is (erroneously) expected to be GET + * + * We will do the incorrect thing (POST-->GET) by default. + * We will provide the capability to do the "right" thing + * (POST-->POST) by a system property, "http.strictPostRedirect=true" + */ + + requests = new MessageHeader(); + setRequests = false; + super.setRequestMethod("GET"); // avoid the connecting check + poster = null; + if (!checkReuseConnection()) + connect(); + + if (!sameDestination(prevURL, url)) { + // Ensures pre-redirect user-set cookie will not be reset. + // CookieHandler, if any, will be queried to determine + // cookies for redirected URL, if any. + userCookies = null; + userCookies2 = null; + } + } else { + if (!checkReuseConnection()) + connect(); + /* Even after a connect() call, http variable still can be + * null, if a ResponseCache has been installed and it returns + * a non-null CacheResponse instance. So check nullity before using it. + * + * And further, if http is null, there's no need to do anything + * about request headers because successive http session will use + * cachedInputStream/cachedHeaders anyway, which is returned by + * CacheResponse. + */ + if (http != null) { + requests.set(0, method + " " + getRequestURI()+" " + + httpVersion, null); + int port = url.getPort(); + String host = stripIPv6ZoneId(url.getHost()); + if (port != -1 && port != url.getDefaultPort()) { + host += ":" + String.valueOf(port); + } + requests.set("Host", host); + } + + if (!sameDestination(prevURL, url)) { + // Redirecting to a different destination will drop any + // security-sensitive headers, regardless of whether + // they are user-set or not. CookieHandler, if any, will be + // queried to determine cookies for redirected URL, if any. + userCookies = null; + userCookies2 = null; + requests.remove("Cookie"); + requests.remove("Cookie2"); + requests.remove("Authorization"); + + // check for preemptive authorization + AuthenticationInfo sauth = + AuthenticationInfo.getServerAuth(url, authCache); + if (sauth != null && sauth.supportsPreemptiveAuthorization() ) { + // Sets "Authorization" + requests.setIfNotSet(sauth.getHeaderName(), sauth.getHeaderValue(url,method)); + currentServerCredentials = sauth; + } + } + } + } + return true; + } + + /* Returns true iff the given URLs have the same host and effective port. */ + private static boolean sameDestination(URL firstURL, URL secondURL) { + assert firstURL.getProtocol().equalsIgnoreCase(secondURL.getProtocol()): + "protocols not equal: " + firstURL + " - " + secondURL; + + if (!firstURL.getHost().equalsIgnoreCase(secondURL.getHost())) + return false; + + int firstPort = firstURL.getPort(); + if (firstPort == -1) + firstPort = firstURL.getDefaultPort(); + int secondPort = secondURL.getPort(); + if (secondPort == -1) + secondPort = secondURL.getDefaultPort(); + if (firstPort != secondPort) + return false; + + return true; + } + + /* dummy byte buffer for reading off socket prior to closing */ + byte[] cdata = new byte [128]; + + /** + * Reset (without disconnecting the TCP conn) in order to do another transaction with this instance + */ + private void reset() throws IOException { + http.reuse = true; + /* must save before calling close */ + reuseClient = http; + InputStream is = http.getInputStream(); + if (!method.equals("HEAD") || tunnelState == TunnelState.SETUP) { + try { + /* we want to read the rest of the response without using the + * hurry mechanism, because that would close the connection + * if everything is not available immediately + */ + if ((is instanceof ChunkedInputStream) || + (is instanceof MeteredStream)) { + /* reading until eof will not block */ + while (is.read (cdata) > 0) {} + } else { + /* raw stream, which will block on read, so only read + * the expected number of bytes, probably 0 + */ + long cl = 0; + int n = 0; + String cls = responses.findValue ("Content-Length"); + if (cls != null) { + try { + cl = Long.parseLong (cls); + } catch (NumberFormatException e) { + cl = 0; + } + } + for (long i=0; i cookies = access.parse(value); + for (HttpCookie cookie : cookies) { + // skip HttpOnly cookies + if (!cookie.isHttpOnly()) + retValue.add(access.header(cookie)); + } + return retValue.toString(); + } + + return value; + } + + // Cache the filtered response headers so that they don't need + // to be generated for every getHeaderFields() call. + private Map> filteredHeaders; // null + + private Map> getFilteredHeaderFields() { + if (filteredHeaders != null) + return filteredHeaders; + + Map> headers, tmpMap = new HashMap<>(); + + if (cachedHeaders != null) + headers = cachedHeaders.getHeaders(); + else + headers = responses.getHeaders(); + + for (Map.Entry> e: headers.entrySet()) { + String key = e.getKey(); + List values = e.getValue(), filteredVals = new ArrayList<>(); + for (String value : values) { + String fVal = filterHeaderField(key, value); + if (fVal != null) + filteredVals.add(fVal); + } + if (!filteredVals.isEmpty()) + tmpMap.put(key, Collections.unmodifiableList(filteredVals)); + } + + return filteredHeaders = Collections.unmodifiableMap(tmpMap); + } + + /** + * Gets a header field by name. Returns null if not known. + * @param name the name of the header field + */ + @Override + public String getHeaderField(String name) { + try { + getInputStream(); + } catch (IOException e) {} + + if (cachedHeaders != null) { + return filterHeaderField(name, cachedHeaders.findValue(name)); + } + + return filterHeaderField(name, responses.findValue(name)); + } + + /** + * Returns an unmodifiable Map of the header fields. + * The Map keys are Strings that represent the + * response-header field names. Each Map value is an + * unmodifiable List of Strings that represents + * the corresponding field values. + * + * @return a Map of header fields + * @since 1.4 + */ + @Override + public Map> getHeaderFields() { + try { + getInputStream(); + } catch (IOException e) {} + + return getFilteredHeaderFields(); + } + + /** + * Gets a header field by index. Returns null if not known. + * @param n the index of the header field + */ + @Override + public String getHeaderField(int n) { + try { + getInputStream(); + } catch (IOException e) {} + + if (cachedHeaders != null) { + return filterHeaderField(cachedHeaders.getKey(n), + cachedHeaders.getValue(n)); + } + return filterHeaderField(responses.getKey(n), responses.getValue(n)); + } + + /** + * Gets a header field by index. Returns null if not known. + * @param n the index of the header field + */ + @Override + public String getHeaderFieldKey(int n) { + try { + getInputStream(); + } catch (IOException e) {} + + if (cachedHeaders != null) { + return cachedHeaders.getKey(n); + } + + return responses.getKey(n); + } + + /** + * Sets request property. If a property with the key already + * exists, overwrite its value with the new value. + * @param value the value to be set + */ + @Override + public void setRequestProperty(String key, String value) { + lock(); + try { + if (connected || connecting) + throw new IllegalStateException("Already connected"); + if (key == null) + throw new NullPointerException("key is null"); + + if (isExternalMessageHeaderAllowed(key, value)) { + requests.set(key, value); + if (!key.equalsIgnoreCase("Content-Type")) { + userHeaders.set(key, value); + } + } + } finally { + unlock(); + } + } + + MessageHeader getUserSetHeaders() { + return userHeaders; + } + + /** + * Adds a general request property specified by a + * key-value pair. This method will not overwrite + * existing values associated with the same key. + * + * @param key the keyword by which the request is known + * (e.g., "accept"). + * @param value the value associated with it. + * @see #getRequestProperty(java.lang.String) + * @since 1.4 + */ + @Override + public void addRequestProperty(String key, String value) { + lock(); + try { + if (connected || connecting) + throw new IllegalStateException("Already connected"); + if (key == null) + throw new NullPointerException("key is null"); + + if (isExternalMessageHeaderAllowed(key, value)) { + requests.add(key, value); + if (!key.equalsIgnoreCase("Content-Type")) { + userHeaders.add(key, value); + } + } + } finally { + unlock(); + } + } + + // + // Set a property for authentication. This can safely disregard + // the connected test. + // + public void setAuthenticationProperty(String key, String value) { + // Only called by the implementation of AuthenticationInfo::setHeaders(...) + // in AuthenticationInfo subclasses, which is only called from + // methods from HttpURLConnection protected by the connectionLock. + assert isLockHeldByCurrentThread(); + + checkMessageHeader(key, value); + requests.set(key, value); + } + + @Override + public String getRequestProperty (String key) { + lock(); + try { + if (key == null) { + return null; + } + + // don't return headers containing security sensitive information + for (int i = 0; i < EXCLUDE_HEADERS.length; i++) { + if (key.equalsIgnoreCase(EXCLUDE_HEADERS[i])) { + return null; + } + } + if (!setUserCookies) { + if (key.equalsIgnoreCase("Cookie")) { + return userCookies; + } + if (key.equalsIgnoreCase("Cookie2")) { + return userCookies2; + } + } + return requests.findValue(key); + } finally { + unlock(); + } + } + + /** + * Returns an unmodifiable Map of general request + * properties for this connection. The Map keys + * are Strings that represent the request-header + * field names. Each Map value is a unmodifiable List + * of Strings that represents the corresponding + * field values. + * + * @return a Map of the general request properties for this connection. + * @throws IllegalStateException if already connected + * @since 1.4 + */ + @Override + public Map> getRequestProperties() { + lock(); + try { + if (connected) + throw new IllegalStateException("Already connected"); + + // exclude headers containing security-sensitive info + if (setUserCookies) { + return requests.getHeaders(EXCLUDE_HEADERS); + } + /* + * The cookies in the requests message headers may have + * been modified. Use the saved user cookies instead. + */ + Map> userCookiesMap = null; + if (userCookies != null || userCookies2 != null) { + userCookiesMap = new HashMap<>(); + if (userCookies != null) { + userCookiesMap.put("Cookie", Arrays.asList(userCookies)); + } + if (userCookies2 != null) { + userCookiesMap.put("Cookie2", Arrays.asList(userCookies2)); + } + } + return requests.filterAndAddHeaders(EXCLUDE_HEADERS2, userCookiesMap); + } finally { + unlock(); + } + } + + @Override + public void setConnectTimeout(int timeout) { + if (timeout < 0) + throw new IllegalArgumentException("timeouts can't be negative"); + connectTimeout = timeout; + } + + + /** + * Returns setting for connect timeout. + *

+ * 0 return implies that the option is disabled + * (i.e., timeout of infinity). + * + * @return an int that indicates the connect timeout + * value in milliseconds + * @see java.net.URLConnection#setConnectTimeout(int) + * @see java.net.URLConnection#connect() + * @since 1.5 + */ + @Override + public int getConnectTimeout() { + return (connectTimeout < 0 ? 0 : connectTimeout); + } + + /** + * Sets the read timeout to a specified timeout, in + * milliseconds. A non-zero value specifies the timeout when + * reading from Input stream when a connection is established to a + * resource. If the timeout expires before there is data available + * for read, a java.net.SocketTimeoutException is raised. A + * timeout of zero is interpreted as an infinite timeout. + * + *

Some non-standard implementation of this method ignores the + * specified timeout. To see the read timeout set, please call + * getReadTimeout(). + * + * @param timeout an int that specifies the timeout + * value to be used in milliseconds + * @throws IllegalArgumentException if the timeout parameter is negative + * + * @see java.net.URLConnection#getReadTimeout() + * @see java.io.InputStream#read() + * @since 1.5 + */ + @Override + public void setReadTimeout(int timeout) { + if (timeout < 0) + throw new IllegalArgumentException("timeouts can't be negative"); + readTimeout = timeout; + } + + /** + * Returns setting for read timeout. 0 return implies that the + * option is disabled (i.e., timeout of infinity). + * + * @return an int that indicates the read timeout + * value in milliseconds + * + * @see java.net.URLConnection#setReadTimeout(int) + * @see java.io.InputStream#read() + * @since 1.5 + */ + @Override + public int getReadTimeout() { + return readTimeout < 0 ? 0 : readTimeout; + } + + public CookieHandler getCookieHandler() { + return cookieHandler; + } + + String getMethod() { + return method; + } + + private MessageHeader mapToMessageHeader(Map> map) { + MessageHeader headers = new MessageHeader(); + if (map == null || map.isEmpty()) { + return headers; + } + for (Map.Entry> entry : map.entrySet()) { + String key = entry.getKey(); + List values = entry.getValue(); + for (String value : values) { + if (key == null) { + headers.prepend(key, value); + } else { + headers.add(key, value); + } + } + } + return headers; + } + + /** + * Returns the given host, without the IPv6 Zone Id, if present. + * (e.g. [fe80::a00:27ff:aaaa:aaaa%eth0] -> [fe80::a00:27ff:aaaa:aaaa]) + * + * @param host host address (not null, not empty) + * @return host address without Zone Id + */ + static String stripIPv6ZoneId(String host) { + if (host.charAt(0) != '[') { // not an IPv6-literal + return host; + } + int i = host.lastIndexOf('%'); + if (i == -1) { // doesn't contain zone_id + return host; + } + return host.substring(0, i) + "]"; + } + + /* The purpose of this wrapper is just to capture the close() call + * so we can check authentication information that may have + * arrived in a Trailer field + */ + class HttpInputStream extends FilterInputStream { + private CacheRequest cacheRequest; + private OutputStream outputStream; + private boolean marked = false; + private int inCache = 0; + private int markCount = 0; + private boolean closed; // false + + public HttpInputStream (InputStream is) { + super (is); + this.cacheRequest = null; + this.outputStream = null; + } + + public HttpInputStream (InputStream is, CacheRequest cacheRequest) { + super (is); + this.cacheRequest = cacheRequest; + try { + this.outputStream = cacheRequest.getBody(); + } catch (IOException ioex) { + this.cacheRequest.abort(); + this.cacheRequest = null; + this.outputStream = null; + } + } + + /** + * Marks the current position in this input stream. A subsequent + * call to the reset method repositions this stream at + * the last marked position so that subsequent reads re-read the same + * bytes. + *

+ * The readlimit argument tells this input stream to + * allow that many bytes to be read before the mark position gets + * invalidated. + *

+ * This method simply performs in.mark(readlimit). + * + * @param readlimit the maximum limit of bytes that can be read before + * the mark position becomes invalid. + * @see java.io.FilterInputStream#in + * @see java.io.FilterInputStream#reset() + */ + // safe to use synchronized here: super method is synchronized too + // and involves no blocking operation; only mark & reset are + // synchronized in the super class hierarchy. + @Override + public synchronized void mark(int readlimit) { + super.mark(readlimit); + if (cacheRequest != null) { + marked = true; + markCount = 0; + } + } + + /** + * Repositions this stream to the position at the time the + * mark method was last called on this input stream. + *

+ * This method + * simply performs in.reset(). + *

+ * Stream marks are intended to be used in + * situations where you need to read ahead a little to see what's in + * the stream. Often this is most easily done by invoking some + * general parser. If the stream is of the type handled by the + * parse, it just chugs along happily. If the stream is not of + * that type, the parser should toss an exception when it fails. + * If this happens within readlimit bytes, it allows the outer + * code to reset the stream and try another parser. + * + * @exception IOException if the stream has not been marked or if the + * mark has been invalidated. + * @see java.io.FilterInputStream#in + * @see java.io.FilterInputStream#mark(int) + */ + // safe to use synchronized here: super method is synchronized too + // and involves no blocking operation; only mark & reset are + // synchronized in the super class hierarchy. + @Override + public synchronized void reset() throws IOException { + super.reset(); + if (cacheRequest != null) { + marked = false; + inCache += markCount; + } + } + + private void ensureOpen() throws IOException { + if (closed) + throw new IOException("stream is closed"); + } + + @Override + public int read() throws IOException { + ensureOpen(); + try { + byte[] b = new byte[1]; + int ret = read(b); + return (ret == -1? ret : (b[0] & 0x00FF)); + } catch (IOException ioex) { + if (cacheRequest != null) { + cacheRequest.abort(); + } + throw ioex; + } + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + try { + int newLen = super.read(b, off, len); + int nWrite; + // write to cache + if (inCache > 0) { + if (inCache >= newLen) { + inCache -= newLen; + nWrite = 0; + } else { + nWrite = newLen - inCache; + inCache = 0; + } + } else { + nWrite = newLen; + } + if (nWrite > 0 && outputStream != null) + outputStream.write(b, off + (newLen-nWrite), nWrite); + if (marked) { + markCount += newLen; + } + return newLen; + } catch (IOException ioex) { + if (cacheRequest != null) { + cacheRequest.abort(); + } + throw ioex; + } + } + + /* skip() calls read() in order to ensure that entire response gets + * cached. same implementation as InputStream.skip */ + + private byte[] skipBuffer; + private static final int SKIP_BUFFER_SIZE = 8096; + + @Override + public long skip (long n) throws IOException { + ensureOpen(); + long remaining = n; + int nr; + if (skipBuffer == null) + skipBuffer = new byte[SKIP_BUFFER_SIZE]; + + byte[] localSkipBuffer = skipBuffer; + + if (n <= 0) { + return 0; + } + + while (remaining > 0) { + nr = read(localSkipBuffer, 0, + (int) Math.min(SKIP_BUFFER_SIZE, remaining)); + if (nr < 0) { + break; + } + remaining -= nr; + } + + return n - remaining; + } + + @Override + public void close () throws IOException { + if (closed) + return; + + try { + if (outputStream != null) { + if (read() != -1) { + cacheRequest.abort(); + } else { + outputStream.close(); + } + } + super.close (); + } catch (IOException ioex) { + if (cacheRequest != null) { + cacheRequest.abort(); + } + throw ioex; + } finally { + closed = true; + HttpURLConnection.this.http = null; + checkResponseCredentials (true); + } + } + } + + class StreamingOutputStream extends FilterOutputStream { + + long expected; + long written; + boolean closed; + boolean error; + IOException errorExcp; + + /** + * expectedLength == -1 if the stream is chunked + * expectedLength > 0 if the stream is fixed content-length + * In the 2nd case, we make sure the expected number + * of bytes are actually written + */ + StreamingOutputStream (OutputStream os, long expectedLength) { + super (os); + expected = expectedLength; + written = 0L; + closed = false; + error = false; + } + + @Override + public void write (int b) throws IOException { + checkError(); + written ++; + if (expected != -1L && written > expected) { + throw new IOException ("too many bytes written"); + } + out.write (b); + } + + @Override + public void write (byte[] b) throws IOException { + write (b, 0, b.length); + } + + @Override + public void write (byte[] b, int off, int len) throws IOException { + checkError(); + written += len; + if (expected != -1L && written > expected) { + out.close (); + throw new IOException ("too many bytes written"); + } + out.write (b, off, len); + } + + void checkError () throws IOException { + if (closed) { + throw new IOException ("Stream is closed"); + } + if (error) { + throw errorExcp; + } + if (out instanceof PrintStream) { + if (((PrintStream) out).checkError()) { + throw new IOException("Error writing request body to server"); + } + } else if (out instanceof ChunkedOutputStream) { + if (((ChunkedOutputStream) out).checkError()) { + throw new IOException("Error writing request body to server"); + } + } + } + + /* this is called to check that all the bytes + * that were supposed to be written were written + * and that the stream is now closed(). + */ + boolean writtenOK () { + return closed && ! error; + } + + @Override + public void close () throws IOException { + if (closed) { + return; + } + closed = true; + if (expected != -1L) { + /* not chunked */ + if (written != expected) { + error = true; + errorExcp = new IOException ("insufficient data written"); + out.close (); + throw errorExcp; + } + super.flush(); /* can't close the socket */ + } else { + /* chunked */ + super.close (); /* force final chunk to be written */ + /* trailing \r\n */ + OutputStream o = http.getOutputStream(); + o.write ('\r'); + o.write ('\n'); + o.flush(); + } + } + } + + + static class ErrorStream extends InputStream { + ByteBuffer buffer; + InputStream is; + + private ErrorStream(ByteBuffer buf) { + buffer = buf; + is = null; + } + + private ErrorStream(ByteBuffer buf, InputStream is) { + buffer = buf; + this.is = is; + } + + // when this method is called, it's either the case that cl > 0, or + // if chunk-encoded, cl = -1; in other words, cl can't be 0 + public static InputStream getErrorStream(InputStream is, long cl, HttpClient http) { + + // cl can't be 0; this following is here for extra precaution + if (cl == 0) { + return null; + } + + try { + // set SO_TIMEOUT to 1/5th of the total timeout + // remember the old timeout value so that we can restore it + int oldTimeout = http.getReadTimeout(); + http.setReadTimeout(timeout4ESBuffer/5); + + long expected = 0; + boolean isChunked = false; + // the chunked case + if (cl < 0) { + expected = bufSize4ES; + isChunked = true; + } else { + expected = cl; + } + if (expected <= bufSize4ES) { + int exp = (int) expected; + byte[] buffer = new byte[exp]; + int count = 0, time = 0, len = 0; + do { + try { + len = is.read(buffer, count, + buffer.length - count); + if (len < 0) { + if (isChunked) { + // chunked ended + // if chunked ended prematurely, + // an IOException would be thrown + break; + } + // the server sends less than cl bytes of data + throw new IOException("the server closes"+ + " before sending "+cl+ + " bytes of data"); + } + count += len; + } catch (SocketTimeoutException ex) { + time += timeout4ESBuffer/5; + } + } while (count < exp && time < timeout4ESBuffer); + + // reset SO_TIMEOUT to old value + http.setReadTimeout(oldTimeout); + + // if count < cl at this point, we will not try to reuse + // the connection + if (count == 0) { + // since we haven't read anything, + // we will return the underlying + // inputstream back to the application + return null; + } else if ((count == expected && !(isChunked)) || (isChunked && len <0)) { + // put the connection into keep-alive cache + // the inputstream will try to do the right thing + is.close(); + return new ErrorStream(ByteBuffer.wrap(buffer, 0, count)); + } else { + // we read part of the response body + return new ErrorStream( + ByteBuffer.wrap(buffer, 0, count), is); + } + } + return null; + } catch (IOException ioex) { + // ioex.printStackTrace(); + return null; + } + } + + @Override + public int available() throws IOException { + if (is == null) { + return buffer.remaining(); + } else { + return buffer.remaining()+is.available(); + } + } + + public int read() throws IOException { + byte[] b = new byte[1]; + int ret = read(b); + return (ret == -1? ret : (b[0] & 0x00FF)); + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int rem = buffer.remaining(); + if (rem > 0) { + int ret = rem < len? rem : len; + buffer.get(b, off, ret); + return ret; + } else { + if (is == null) { + return -1; + } else { + return is.read(b, off, len); + } + } + } + + @Override + public void close() throws IOException { + buffer = null; + if (is != null) { + is.close(); + } + } + } + + @SuppressWarnings("deprecation") + private static URL newURL(String spec) throws MalformedURLException { + return new URL(spec); + } + + @SuppressWarnings("deprecation") + private static URL newURL(URL context, String spec) throws MalformedURLException { + return new URL(context, spec); + } + + // ensure there are no null characters in username or password + private static void validateNTLMCredentials(PasswordAuthentication pw) + throws IOException { + + if (pw == null) { + return; + } + char[] password = pw.getPassword(); + if (password != null) { + for (int i=0; i Date: Wed, 11 Nov 2026 13:52:24 +0000 Subject: [PATCH 19/19] Remove a file added by mistake --- jdkhttp.java | 3779 -------------------------------------------------- 1 file changed, 3779 deletions(-) delete mode 100644 jdkhttp.java diff --git a/jdkhttp.java b/jdkhttp.java deleted file mode 100644 index 7b750642..00000000 --- a/jdkhttp.java +++ /dev/null @@ -1,3779 +0,0 @@ -/* - * Copyright (c) 1995, 2026, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package sun.net.www.protocol.http; - -import java.util.Arrays; -import java.net.URL; -import java.net.URLConnection; -import java.net.ProtocolException; -import java.net.HttpRetryException; -import java.net.PasswordAuthentication; -import java.net.Authenticator; -import java.net.HttpCookie; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.net.SocketTimeoutException; -import java.net.Proxy; -import java.net.ProxySelector; -import java.net.URI; -import java.net.InetSocketAddress; -import java.net.CookieHandler; -import java.net.ResponseCache; -import java.net.CacheResponse; -import java.net.SecureCacheResponse; -import java.net.CacheRequest; -import java.net.Authenticator.RequestorType; -import java.io.*; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.Map; -import java.util.List; -import java.util.Locale; -import java.util.StringTokenizer; -import java.util.Iterator; -import java.util.HashSet; -import java.util.HashMap; -import java.util.Set; -import java.util.StringJoiner; -import jdk.internal.access.JavaNetHttpCookieAccess; -import jdk.internal.access.SharedSecrets; -import jdk.internal.util.Exceptions; -import sun.net.NetProperties; -import sun.net.NetworkClient; -import sun.net.util.IPAddressUtil; -import sun.net.www.*; -import sun.net.www.http.HttpClient; -import sun.net.www.http.PosterOutputStream; -import sun.net.www.http.ChunkedInputStream; -import sun.net.www.http.ChunkedOutputStream; -import sun.util.logging.PlatformLogger; -import java.text.SimpleDateFormat; -import java.util.TimeZone; -import java.net.MalformedURLException; -import java.nio.ByteBuffer; -import java.util.Objects; -import java.util.concurrent.locks.ReentrantLock; - -import static sun.net.util.ProxyUtil.copyProxy; -import static sun.net.www.protocol.http.AuthScheme.BASIC; -import static sun.net.www.protocol.http.AuthScheme.DIGEST; -import static sun.net.www.protocol.http.AuthScheme.NTLM; -import static sun.net.www.protocol.http.AuthScheme.NEGOTIATE; -import static sun.net.www.protocol.http.AuthScheme.KERBEROS; -import static sun.net.www.protocol.http.AuthScheme.UNKNOWN; - -/** - * A class to represent an HTTP connection to a remote object. - */ - - -public class HttpURLConnection extends java.net.HttpURLConnection { - - static final String HTTP_CONNECT = "CONNECT"; - - static final String version; - public static final String userAgent; - - /* max # of allowed re-directs */ - static final int defaultmaxRedirects = 20; - static final int maxRedirects; - - /* Not all servers support the (Proxy)-Authentication-Info headers. - * By default, we don't require them to be sent - */ - static final boolean validateProxy; - static final boolean validateServer; - - /** A, possibly empty, set of authentication schemes that are disabled - * when proxying plain HTTP ( not HTTPS ). */ - static final Set disabledProxyingSchemes; - - /** A, possibly empty, set of authentication schemes that are disabled - * when setting up a tunnel for HTTPS ( HTTP CONNECT ). */ - static final Set disabledTunnelingSchemes; - - private StreamingOutputStream strOutputStream; - private static final String RETRY_MSG1 = - "cannot retry due to proxy authentication, in streaming mode"; - private static final String RETRY_MSG2 = - "cannot retry due to server authentication, in streaming mode"; - private static final String RETRY_MSG3 = - "cannot retry due to redirection, in streaming mode"; - - /* - * System properties related to error stream handling: - * - * sun.net.http.errorstream.enableBuffering = - * - * With the above system property set to true (default is false), - * when the response code is >=400, the HTTP handler will try to - * buffer the response body (up to a certain amount and within a - * time limit). Thus freeing up the underlying socket connection - * for reuse. The rationale behind this is that usually when the - * server responds with a >=400 error (client error or server - * error, such as 404 file not found), the server will send a - * small response body to explain who to contact and what to do to - * recover. With this property set to true, even if the - * application doesn't call getErrorStream(), read the response - * body, and then call close(), the underlying socket connection - * can still be kept-alive and reused. The following two system - * properties provide further control to the error stream - * buffering behaviour. - * - * sun.net.http.errorstream.timeout = - * the timeout (in millisec) waiting the error stream - * to be buffered; default is 300 ms - * - * sun.net.http.errorstream.bufferSize = - * the size (in bytes) to use for the buffering the error stream; - * default is 4k - */ - - - /* Should we enable buffering of error streams? */ - private static final boolean enableESBuffer; - - /* timeout waiting for read for buffered error stream; - */ - private static final int timeout4ESBuffer; - - /* buffer size for buffered error stream; - */ - private static final int bufSize4ES; - - private static final int maxHeaderSize; - - /* - * Restrict setting of request headers through the public api - * consistent with JavaScript XMLHttpRequest2 with a few - * exceptions. Disallowed headers are silently ignored for - * backwards compatibility reasons. For example, some applets set the - * Host header since old JREs did not implement HTTP 1.1. - * Additionally, any header starting with Sec- is - * disallowed. - * - * The following headers are allowed for historical reasons: - * - * Accept-Charset, Accept-Encoding, Cookie, Cookie2, Date, - * Referer, TE, User-Agent, headers beginning with Proxy-. - * - * The following headers are allowed in a limited form: - * - * Connection: close - * - * See http://www.w3.org/TR/XMLHttpRequest2. - */ - private static final boolean allowRestrictedHeaders; - private static final Set restrictedHeaderSet; - private static final String[] restrictedHeaders = { - /* Restricted by XMLHttpRequest2 */ - //"Accept-Charset", - //"Accept-Encoding", - "Access-Control-Request-Headers", - "Access-Control-Request-Method", - "Connection", /* close is allowed */ - "Content-Length", - //"Cookie", - //"Cookie2", - "Content-Transfer-Encoding", - //"Date", - //"Expect", - "Host", - "Keep-Alive", - "Origin", - // "Referer", - // "TE", - "Trailer", - "Transfer-Encoding", - "Upgrade", - //"User-Agent", - "Via" - }; - - private static Set schemesListToSet(String list) { - if (list == null || list.isEmpty()) - return Collections.emptySet(); - - Set s = new HashSet<>(); - String[] parts = list.split("\\s*,\\s*"); - for (String part : parts) - s.add(part.toLowerCase(Locale.ROOT)); - return s; - } - - static { - maxRedirects = Integer.getInteger("http.maxRedirects", defaultmaxRedirects); - version = System.getProperty("java.version"); - String agent = System.getProperty("http.agent"); - if (agent == null) { - agent = "Java/"+version; - } else { - agent = agent + " Java/"+version; - } - userAgent = agent; - - // A set of net properties to control the use of authentication schemes - // when proxying/tunneling. - String p = NetProperties.get("jdk.http.auth.tunneling.disabledSchemes"); - disabledTunnelingSchemes = schemesListToSet(p); - p = NetProperties.get("jdk.http.auth.proxying.disabledSchemes"); - disabledProxyingSchemes = schemesListToSet(p); - - validateProxy = Boolean.getBoolean("http.auth.digest.validateProxy"); - validateServer = Boolean.getBoolean("http.auth.digest.validateServer"); - - enableESBuffer = Boolean.getBoolean("sun.net.http.errorstream.enableBuffering"); - int esBufferTimeout = Integer.getInteger( - "sun.net.http.errorstream.timeout", 300); - if (esBufferTimeout <= 0) { - esBufferTimeout = 300; // use the default - } - timeout4ESBuffer = esBufferTimeout; - - int esBufSize = Integer.getInteger( - "sun.net.http.errorstream.bufferSize", 4096); - if (esBufSize <= 0) { - esBufSize = 4096; // use the default - } - bufSize4ES = esBufSize; - - allowRestrictedHeaders = Boolean.getBoolean("sun.net.http.allowRestrictedHeaders"); - if (!allowRestrictedHeaders) { - restrictedHeaderSet = HashSet.newHashSet(restrictedHeaders.length); - for (int i=0; i < restrictedHeaders.length; i++) { - restrictedHeaderSet.add(restrictedHeaders[i].toLowerCase(Locale.ROOT)); - } - } else { - restrictedHeaderSet = null; - } - - int defMaxHeaderSize = 384 * 1024; - String maxHeaderSizeStr = NetProperties.get("jdk.http.maxHeaderSize"); - int maxHeaderSizeVal = defMaxHeaderSize; - if (maxHeaderSizeStr != null) { - try { - maxHeaderSizeVal = Integer.parseInt(maxHeaderSizeStr); - } catch (NumberFormatException n) { - maxHeaderSizeVal = defMaxHeaderSize; - } - } - if (maxHeaderSizeVal < 0) maxHeaderSizeVal = 0; - maxHeaderSize = maxHeaderSizeVal; - } - - static final String httpVersion = "HTTP/1.1"; - static final String acceptString = "*/*"; - - // the following http request headers should NOT have their values - // returned for security reasons. - private static final String[] EXCLUDE_HEADERS = { - "Proxy-Authorization", - "Authorization" - }; - - // also exclude system cookies when any might be set - private static final String[] EXCLUDE_HEADERS2= { - "Proxy-Authorization", - "Authorization", - "Cookie", - "Cookie2" - }; - - protected HttpClient http; - protected Handler handler; - protected Proxy instProxy; - protected volatile Authenticator authenticator; - protected volatile AuthCacheImpl authCache = AuthCacheImpl.getDefault(); - - private CookieHandler cookieHandler; - private final ResponseCache cacheHandler; - - private volatile boolean usingProxy; - - // the cached response, and cached response headers and body - protected CacheResponse cachedResponse; - private MessageHeader cachedHeaders; - private InputStream cachedInputStream; - - /* output stream to server */ - protected PrintStream ps = null; - - /* buffered error stream */ - private InputStream errorStream = null; - - /* User set Cookies */ - private boolean setUserCookies = true; - private String userCookies = null; - private String userCookies2 = null; - - /* all the headers we send - * NOTE: do *NOT* dump out the content of 'requests' in the - * output or stacktrace since it may contain security-sensitive - * headers such as those defined in EXCLUDE_HEADERS. - */ - private MessageHeader requests; - - /* The headers actually set by the user are recorded here also - */ - private final MessageHeader userHeaders; - - /* Headers and request method cannot be changed - * once this flag is set in :- - * - getOutputStream() - * - getInputStream()) - * - connect() - * Access is protected by connectionLock. - */ - private boolean connecting = false; - - /* The following two fields are only used with Digest Authentication */ - String domain; /* The list of authentication domains */ - DigestAuthentication.Parameters digestparams; - - /* Current credentials in use */ - AuthenticationInfo currentProxyCredentials = null; - AuthenticationInfo currentServerCredentials = null; - boolean needToCheck = true; - private boolean doingNTLM2ndStage = false; /* doing the 2nd stage of an NTLM server authentication */ - private boolean doingNTLMp2ndStage = false; /* doing the 2nd stage of an NTLM proxy authentication */ - - /* try auth without calling Authenticator. Used for transparent NTLM authentication */ - private boolean tryTransparentNTLMServer = true; - private boolean tryTransparentNTLMProxy = true; - private boolean useProxyResponseCode = false; - - // used when redirecting to compare current and previous proxies - private Proxy lastProxy; - - /* Used by Windows specific code */ - private Object authObj; - - /* Set if the user is manually setting the Authorization or Proxy-Authorization headers */ - boolean isUserServerAuth; - boolean isUserProxyAuth; - - String serverAuthKey, proxyAuthKey; - - /* all the response headers we get back */ - private MessageHeader responses; - /* the stream _from_ the server */ - private InputStream inputStream = null; - /* post stream _to_ the server, if any */ - private PosterOutputStream poster = null; - - /* Indicates if the std. request headers have been set in requests. */ - private boolean setRequests=false; - - /* Indicates whether a request has already failed or not */ - private boolean failedOnce=false; - - /* Remembered Exception, we will throw it again if somebody - calls getInputStream after disconnect */ - private Exception rememberedException = null; - - /* Remembered Exception, we will throw it again if somebody - calls getOutputStream after disconnect or error */ - private Exception rememberedExceptionOut = null; - - /* If we decide we want to reuse a client, we put it here */ - private HttpClient reuseClient = null; - - /* Tunnel states */ - public enum TunnelState { - /* No tunnel */ - NONE, - - /* Setting up a tunnel */ - SETUP, - - /* Tunnel has been successfully setup */ - TUNNELING - } - - private TunnelState tunnelState = TunnelState.NONE; - - /* Redefine timeouts from java.net.URLConnection as we need -1 to mean - * not set. This is to ensure backward compatibility. - */ - private int connectTimeout = NetworkClient.DEFAULT_CONNECT_TIMEOUT; - private int readTimeout = NetworkClient.DEFAULT_READ_TIMEOUT; - - /* Logging support */ - private static final PlatformLogger logger = - PlatformLogger.getLogger("sun.net.www.protocol.http.HttpURLConnection"); - - /* Lock */ - private final ReentrantLock connectionLock = new ReentrantLock(); - - private final void lock() { - connectionLock.lock(); - } - - private final void unlock() { - connectionLock.unlock(); - } - - public final boolean isLockHeldByCurrentThread() { - return connectionLock.isHeldByCurrentThread(); - } - - /* - * Calls Authenticator.requestPasswordAuthentication - * - */ - private static PasswordAuthentication requestPassword( - final Authenticator authenticator, - final String host, - final InetAddress addr, - final int port, - final String protocol, - final String prompt, - final String scheme, - final URL url, - final RequestorType authType) { - - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - logger.finest("Requesting Authentication: host =" + host + " url = " + url); - } - PasswordAuthentication pass = Authenticator.requestPasswordAuthentication( - authenticator, host, addr, port, protocol, - prompt, scheme, url, authType); - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - logger.finest("Authentication returned: " + (pass != null ? pass.toString() : "null")); - } - return pass; - } - - private boolean isRestrictedHeader(String key, String value) { - if (allowRestrictedHeaders) { - return false; - } - - key = key.toLowerCase(Locale.ROOT); - if (restrictedHeaderSet.contains(key)) { - /* - * Exceptions to restricted headers: - * - * Allow "Connection: close". - */ - if (key.equals("connection") && value.equalsIgnoreCase("close")) { - return false; - } - return true; - } else if (key.startsWith("sec-")) { - return true; - } - return false; - } - - /* - * Checks the validity of http message header and whether the header - * is restricted and throws IllegalArgumentException if invalid or - * restricted. - */ - private boolean isExternalMessageHeaderAllowed(String key, String value) { - checkMessageHeader(key, value); - if (!isRestrictedHeader(key, value)) { - return true; - } - return false; - } - - /* Logging support */ - public static PlatformLogger getHttpLogger() { - return logger; - } - - /* Used for Windows NTLM implementation */ - public Object authObj() { - return authObj; - } - - public void authObj(Object authObj) { - this.authObj = authObj; - } - - @Override - public void setAuthenticator(Authenticator auth) { - lock(); - try { - if (connecting || connected) { - throw new IllegalStateException( - "Authenticator must be set before connecting"); - } - authenticator = Objects.requireNonNull(auth); - authCache = AuthCacheImpl.getAuthCacheFor(authenticator); - } finally { - unlock(); - } - } - - public AuthCacheImpl getAuthCache() { - return authCache; - } - - /* - * checks the validity of http message header and throws - * IllegalArgumentException if invalid. - */ - private void checkMessageHeader(String key, String value) { - char LF = '\n'; - int index = key.indexOf(LF); - int index1 = key.indexOf(':'); - if (index != -1 || index1 != -1) { - throw new IllegalArgumentException( - "Illegal character(s) in message header field: " + key); - } - else { - if (value == null) { - return; - } - - index = value.indexOf(LF); - while (index != -1) { - index++; - if (index < value.length()) { - char c = value.charAt(index); - if ((c==' ') || (c=='\t')) { - // ok, check the next occurrence - index = value.indexOf(LF, index); - continue; - } - } - throw new IllegalArgumentException( - "Illegal character(s) in message header value: " + value); - } - } - } - - public void setRequestMethod(String method) - throws ProtocolException { - lock(); - try { - if (connected || connecting) { - throw new IllegalStateException("Already connected"); - } - super.setRequestMethod(method); - } finally { - unlock(); - } - } - - /* adds the standard key/val pairs to requests if necessary & write to - * given PrintStream - */ - private void writeRequests() throws IOException { - assert isLockHeldByCurrentThread(); - - /* print all message headers in the MessageHeader - * onto the wire - all the ones we've set and any - * others that have been set - */ - // send any pre-emptive authentication - if (http.usingProxy && tunnelState() != TunnelState.TUNNELING) { - setPreemptiveProxyAuthentication(requests); - } - if (!setRequests) { - - /* We're very particular about the order in which we - * set the request headers here. The order should not - * matter, but some careless CGI programs have been - * written to expect a very particular order of the - * standard headers. To name names, the order in which - * Navigator3.0 sends them. In particular, we make *sure* - * to send Content-type: <> and Content-length:<> second - * to last and last, respectively, in the case of a POST - * request. - */ - final String requestLine = method + " " + getRequestURI()+ " " + httpVersion; - final int requestLineIndex = requests.getKey(requestLine); - if (requestLineIndex != 0) { - // we expect the request line to be at index 0. we set it here - // if we don't find the request line at that index. - requests.prepend(requestLine, null); - } - if (!getUseCaches()) { - requests.setIfNotSet ("Cache-Control", "no-cache"); - requests.setIfNotSet ("Pragma", "no-cache"); - } - requests.setIfNotSet("User-Agent", userAgent); - int port = url.getPort(); - String host = stripIPv6ZoneId(url.getHost()); - if (port != -1 && port != url.getDefaultPort()) { - host += ":" + String.valueOf(port); - } - // if the "Host" header hasn't been explicitly set, then set its - // value to the one determined through the request URL - requests.setIfNotSet("Host", host); - - requests.setIfNotSet("Accept", acceptString); - - /* - * For HTTP/1.1 the default behavior is to keep connections alive. - * However, we may be talking to a 1.0 server so we should set - * keep-alive just in case, except if we have encountered an error - * or if keep alive is disabled via a system property - */ - - if (http.getHttpKeepAliveSet()) { - if (http.usingProxy && tunnelState() != TunnelState.TUNNELING) { - requests.setIfNotSet("Proxy-Connection", "keep-alive"); - } else { - requests.setIfNotSet("Connection", "keep-alive"); - } - } else { - /* - * RFC 2616 HTTP/1.1 section 14.10 says: - * HTTP/1.1 applications that do not support persistent - * connections MUST include the "close" connection option - * in every message - */ - requests.setIfNotSet("Connection", "close"); - } - // Set modified since if necessary - long modTime = getIfModifiedSince(); - if (modTime != 0 ) { - Date date = new Date(modTime); - //use the preferred date format according to RFC 2068(HTTP1.1), - // RFC 822 and RFC 1123 - SimpleDateFormat fo = - new SimpleDateFormat ("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US); - fo.setTimeZone(TimeZone.getTimeZone("GMT")); - requests.setIfNotSet("If-Modified-Since", fo.format(date)); - } - // check for preemptive authorization - AuthenticationInfo sauth = AuthenticationInfo.getServerAuth(url, authCache); - if (sauth != null && sauth.supportsPreemptiveAuthorization() ) { - // Sets "Authorization" - requests.setIfNotSet(sauth.getHeaderName(), sauth.getHeaderValue(url,method)); - currentServerCredentials = sauth; - } - - if (!method.equals("PUT") && (poster != null || streaming())) { - requests.setIfNotSet ("Content-type", - "application/x-www-form-urlencoded"); - } - - boolean chunked = false; - - if (streaming()) { - if (chunkLength != -1) { - requests.set ("Transfer-Encoding", "chunked"); - chunked = true; - } else { /* fixed content length */ - if (fixedContentLengthLong != -1) { - requests.set ("Content-Length", - String.valueOf(fixedContentLengthLong)); - } else if (fixedContentLength != -1) { - requests.set ("Content-Length", - String.valueOf(fixedContentLength)); - } - } - } else if (poster != null) { - /* add Content-Length & POST/PUT data */ - // safe to synchronize on poster: this is - // a simple subclass of ByteArrayOutputStream - synchronized (poster) { - /* close it, so no more data can be added */ - poster.close(); - requests.set("Content-Length", - String.valueOf(poster.size())); - } - } - - if (!chunked) { - if (requests.findValue("Transfer-Encoding") != null) { - requests.remove("Transfer-Encoding"); - if (logger.isLoggable(PlatformLogger.Level.WARNING)) { - logger.warning( - "use streaming mode for chunked encoding"); - } - } - } - - // get applicable cookies based on the uri and request headers - // add them to the existing request headers - setCookieHeader(); - - setRequests=true; - } - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine(requests.toString()); - } - http.writeRequests(requests, poster, streaming()); - if (ps.checkError()) { - String proxyHost = http.getProxyHostUsed(); - int proxyPort = http.getProxyPortUsed(); - disconnectInternal(); - if (failedOnce) { - throw new IOException("Error writing to server"); - } else { // try once more - failedOnce=true; - if (proxyHost != null) { - setProxiedClient(url, proxyHost, proxyPort); - } else { - setNewClient (url); - } - ps = (PrintStream) http.getOutputStream(); - connected=true; - responses = new MessageHeader(maxHeaderSize); - setRequests=false; - writeRequests(); - } - } - } - - /** - * Create a new HttpClient object, bypassing the cache of - * HTTP client objects/connections. - * - * @param url the URL being accessed - */ - protected void setNewClient (URL url) - throws IOException { - setNewClient(url, false); - } - - /** - * Obtain a HttpsClient object. Use the cached copy if specified. - * - * @param url the URL being accessed - * @param useCache whether the cached connection should be used - * if present - */ - protected void setNewClient (URL url, boolean useCache) - throws IOException { - http = HttpClient.New(url, null, -1, useCache, connectTimeout, this); - http.setReadTimeout(readTimeout); - } - - - /** - * Create a new HttpClient object, set up so that it uses - * per-instance proxying to the given HTTP proxy. This - * bypasses the cache of HTTP client objects/connections. - * - * @param url the URL being accessed - * @param proxyHost the proxy host to use - * @param proxyPort the proxy port to use - */ - protected void setProxiedClient (URL url, String proxyHost, int proxyPort) - throws IOException { - setProxiedClient(url, proxyHost, proxyPort, false); - } - - /** - * Obtain a HttpClient object, set up so that it uses per-instance - * proxying to the given HTTP proxy. Use the cached copy of HTTP - * client objects/connections if specified. - * - * @param url the URL being accessed - * @param proxyHost the proxy host to use - * @param proxyPort the proxy port to use - * @param useCache whether the cached connection should be used - * if present - */ - protected void setProxiedClient (URL url, - String proxyHost, int proxyPort, - boolean useCache) - throws IOException { - proxiedConnect(url, proxyHost, proxyPort, useCache); - } - - protected void proxiedConnect(URL url, - String proxyHost, int proxyPort, - boolean useCache) - throws IOException { - http = HttpClient.New (url, proxyHost, proxyPort, useCache, - connectTimeout, this); - http.setReadTimeout(readTimeout); - } - - protected HttpURLConnection(URL u, Handler handler) - throws IOException { - // we set proxy == null to distinguish this case with the case - // when per connection proxy is set - this(u, null, handler); - } - - private static String checkHost(String h) throws IOException { - if (h != null) { - if (h.indexOf('\n') > -1) { - throw new MalformedURLException("Illegal character in host"); - } - } - return h; - } - public HttpURLConnection(URL u, String host, int port) throws IOException { - this(u, new Proxy(Proxy.Type.HTTP, - InetSocketAddress.createUnresolved(checkHost(host), port))); - } - - /** this constructor is used by other protocol handlers such as ftp - that want to use http to fetch urls on their behalf.*/ - public HttpURLConnection(URL u, Proxy p) throws IOException { - this(u, p, new Handler()); - } - - private static URL checkURL(URL u) throws IOException { - if (u != null) { - if (u.toExternalForm().indexOf('\n') > -1) { - throw new MalformedURLException("Illegal character in URL"); - } - } - String s = IPAddressUtil.checkAuthority(u); - if (s != null) { - throw new MalformedURLException(s); - } - return u; - } - - protected HttpURLConnection(URL u, Proxy p, Handler handler) - throws IOException { - super(checkURL(u)); - requests = new MessageHeader(); - responses = new MessageHeader(maxHeaderSize); - userHeaders = new MessageHeader(); - this.handler = handler; - instProxy = copyProxy(p); - cookieHandler = CookieHandler.getDefault(); - cacheHandler = ResponseCache.getDefault(); - } - - // overridden in HTTPS subclass - - public void connect() throws IOException { - lock(); - try { - connecting = true; - } finally { - unlock(); - } - plainConnect(); - } - - private boolean checkReuseConnection () { - if (connected) { - return true; - } - if (reuseClient != null) { - http = reuseClient; - http.setReadTimeout(getReadTimeout()); - http.reuse = false; - reuseClient = null; - connected = true; - return true; - } - return false; - } - - protected void plainConnect() throws IOException { - lock(); - try { - if (connected) { - return; - } - } finally { - unlock(); - } - plainConnect0(); - } - - protected void plainConnect0() throws IOException { - // try to see if request can be served from local cache - if (cacheHandler != null && getUseCaches()) { - try { - URI uri = ParseUtil.toURI(url); - if (uri != null) { - cachedResponse = cacheHandler.get(uri, getRequestMethod(), getUserSetHeaders().getHeaders()); - if ("https".equalsIgnoreCase(uri.getScheme()) - && !(cachedResponse instanceof SecureCacheResponse)) { - cachedResponse = null; - } - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - logger.finest("Cache Request for " + uri + " / " + getRequestMethod()); - logger.finest("From cache: " + (cachedResponse != null ? cachedResponse.toString() : "null")); - } - if (cachedResponse != null) { - cachedHeaders = mapToMessageHeader(cachedResponse.getHeaders()); - cachedInputStream = cachedResponse.getBody(); - } - } - } catch (IOException ioex) { - // ignore and commence normal connection - } - if (cachedHeaders != null && cachedInputStream != null) { - connected = true; - return; - } else { - cachedResponse = null; - } - } - try { - /* Try to open connections using the following scheme, - * return on the first one that's successful: - * 1) if (instProxy != null) - * connect to instProxy; raise exception if failed - * 2) else use system default ProxySelector - * 3) else make a direct connection if ProxySelector is not present - */ - - if (instProxy == null) { // no instance Proxy is set - /** - * Do we have to use a proxy? - */ - final ProxySelector sel = ProxySelector.getDefault(); - if (sel != null) { - URI uri = sun.net.www.ParseUtil.toURI(url); - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - logger.finest("ProxySelector Request for " + uri); - } - final List proxies; - try { - proxies = sel.select(uri); - } catch (IllegalArgumentException iae) { - throw new IOException("Failed to select a proxy", iae); - } - final Iterator it = proxies.iterator(); - Proxy p; - while (it.hasNext()) { - p = copyProxy(it.next()); - try { - if (!failedOnce) { - http = getNewHttpClient(url, p, connectTimeout); - http.setReadTimeout(readTimeout); - } else { - // make sure to construct new connection if first - // attempt failed - http = getNewHttpClient(url, p, connectTimeout, false); - http.setReadTimeout(readTimeout); - } - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - if (p != null) { - logger.finest("Proxy used: " + p.toString()); - } - } - break; - } catch (IOException ioex) { - if (p != Proxy.NO_PROXY) { - sel.connectFailed(uri, p.address(), ioex); - if (!it.hasNext()) { - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - logger.finest("Retrying with proxy: " + p.toString()); - } - http = getNewHttpClient(url, p, connectTimeout, false); - http.setReadTimeout(readTimeout); - break; - } - } else { - throw ioex; - } - continue; - } - } - } else { - // No proxy selector, create http client with no proxy - if (!failedOnce) { - http = getNewHttpClient(url, null, connectTimeout); - http.setReadTimeout(readTimeout); - } else { - // make sure to construct new connection if first - // attempt failed - http = getNewHttpClient(url, null, connectTimeout, false); - http.setReadTimeout(readTimeout); - } - } - } else { - if (!failedOnce) { - http = getNewHttpClient(url, instProxy, connectTimeout); - http.setReadTimeout(readTimeout); - } else { - // make sure to construct new connection if first - // attempt failed - http = getNewHttpClient(url, instProxy, connectTimeout, false); - http.setReadTimeout(readTimeout); - } - } - - usingProxy = usingProxy || usingProxyInternal(); - ps = (PrintStream)http.getOutputStream(); - } catch (IOException e) { - throw e; - } - // constructor to HTTP client calls openserver - connected = true; - } - - // subclass HttpsClient will overwrite & return an instance of HttpsClient - protected HttpClient getNewHttpClient(URL url, Proxy p, int connectTimeout) - throws IOException { - return HttpClient.New(url, p, connectTimeout, this); - } - - // subclass HttpsClient will overwrite & return an instance of HttpsClient - protected HttpClient getNewHttpClient(URL url, Proxy p, - int connectTimeout, boolean useCache) - throws IOException { - return HttpClient.New(url, p, connectTimeout, useCache, this); - } - - private void expect100Continue() throws IOException { - // Expect: 100-Continue was set, so check the return code for - // Acceptance - int oldTimeout = http.getReadTimeout(); - boolean timedOut = false; - boolean tempTimeOutSet = false; - if (oldTimeout <= 0 || oldTimeout > 5000) { - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine("Timeout currently set to " + - oldTimeout + " temporarily setting it to 5 seconds"); - } - // 5s read timeout in case the server doesn't understand - // Expect: 100-Continue - http.setReadTimeout(5000); - tempTimeOutSet = true; - } - - try { - http.parseHTTP(responses, this); - } catch (SocketTimeoutException se) { - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine("SocketTimeoutException caught," + - " will attempt to send body regardless"); - } - timedOut = true; - } - - if (!timedOut) { - // Can't use getResponseCode() yet - String resp = responses.getValue(0); - // Parse the response which is of the form: - // HTTP/1.1 417 Expectation Failed - // HTTP/1.1 100 Continue - if (resp != null && resp.startsWith("HTTP/")) { - String[] sa = resp.split("\\s+"); - responseCode = -1; - try { - // Response code is 2nd token on the line - if (sa.length > 1) - responseCode = Integer.parseInt(sa[1]); - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine("response code received " + responseCode); - } - if (sa.length > 2) - responseMessage = String.join(" ", Arrays.copyOfRange(sa, 2, sa.length)); - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine("response message received " + responseMessage); - } - } catch (NumberFormatException numberFormatException) { - } - } - if (responseCode != 100) { - // responseCode will be returned to caller - throw new ProtocolException("Server rejected operation"); - } - } - - // If timeout was changed, restore to original value - if (tempTimeOutSet) { - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine("Restoring original timeout : " + oldTimeout); - } - http.setReadTimeout(oldTimeout); - } - - // Ignore any future 100 continue messages - http.setIgnoreContinue(true); - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine("Set Ignore Continue to true"); - } - - responseCode = -1; - responses.reset(); - // Proceed - } - - /* - * Allowable input/output sequences: - * [interpreted as request entity] - * - get output, [write output,] get input, [read input] - * - get output, [write output] - * [interpreted as GET] - * - get input, [read input] - * Disallowed: - * - get input, [read input,] get output, [write output] - */ - - @Override - public OutputStream getOutputStream() throws IOException { - lock(); - try { - connecting = true; - return getOutputStream0(); - } finally { - unlock(); - } - } - - private OutputStream getOutputStream0() throws IOException { - assert isLockHeldByCurrentThread(); - try { - if (!doOutput) { - throw new ProtocolException("cannot write to a URLConnection" - + " if doOutput=false - call setDoOutput(true)"); - } - - if (rememberedExceptionOut != null) { - if (rememberedExceptionOut instanceof RuntimeException) { - throw new RuntimeException(rememberedExceptionOut); - } else { - throw getChainedException((IOException) rememberedExceptionOut); - } - } - - if (method.equals("GET")) { - method = "POST"; // Backward compatibility - } - if ("TRACE".equals(method) && "http".equals(url.getProtocol())) { - throw new ProtocolException("HTTP method TRACE" + - " doesn't support output"); - } - - // if there's already an input stream open, throw an exception - if (inputStream != null) { - throw new ProtocolException("Cannot write output after reading input."); - } - - if (!checkReuseConnection()) - connect(); - - boolean expectContinue = false; - String expects = requests.findValue("Expect"); - if ("100-Continue".equalsIgnoreCase(expects) && streaming()) { - expectContinue = true; - } - - if (streaming() && strOutputStream == null) { - writeRequests(); - } - - if (expectContinue) { - http.setIgnoreContinue(false); - expect100Continue(); - } - ps = (PrintStream)http.getOutputStream(); - if (streaming()) { - if (strOutputStream == null) { - if (chunkLength != -1) { /* chunked */ - strOutputStream = new StreamingOutputStream( - new ChunkedOutputStream(ps, chunkLength), -1L); - } else { /* must be fixed content length */ - long length = 0L; - if (fixedContentLengthLong != -1) { - length = fixedContentLengthLong; - } else if (fixedContentLength != -1) { - length = fixedContentLength; - } - strOutputStream = new StreamingOutputStream(ps, length); - } - } - return strOutputStream; - } else { - if (poster == null) { - poster = new PosterOutputStream(); - } - return poster; - } - } catch (ProtocolException e) { - // Save the response code which may have been set while enforcing - // the 100-continue. disconnectInternal() forces it to -1 - int i = responseCode; - disconnectInternal(); - responseCode = i; - rememberedExceptionOut = e; - throw e; - } catch (RuntimeException | IOException e) { - disconnectInternal(); - rememberedExceptionOut = e; - throw e; - } - } - - // Streaming returns true if there is a request body to send - public boolean streaming () { - return (fixedContentLength != -1) || (fixedContentLengthLong != -1) || - (chunkLength != -1); - } - - /* - * get applicable cookies based on the uri and request headers - * add them to the existing request headers - */ - private void setCookieHeader() throws IOException { - if (cookieHandler != null) { - // we only want to capture the user defined Cookies once, as - // they cannot be changed by user code after we are connected, - // only internally. - - // we should only reach here when called from - // writeRequest, which in turn is only called by - // getInputStream0 - assert isLockHeldByCurrentThread(); - if (setUserCookies) { - int k = requests.getKey("Cookie"); - if (k != -1) - userCookies = requests.getValue(k); - k = requests.getKey("Cookie2"); - if (k != -1) - userCookies2 = requests.getValue(k); - setUserCookies = false; - } - - // remove old Cookie header before setting new one. - requests.remove("Cookie"); - requests.remove("Cookie2"); - - URI uri = ParseUtil.toURI(url); - if (uri != null) { - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - logger.finest("CookieHandler request for " + uri); - } - Map> cookies - = cookieHandler.get( - uri, requests.getHeaders(EXCLUDE_HEADERS)); - if (!cookies.isEmpty()) { - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - logger.finest("Cookies retrieved: " + cookies.toString()); - } - for (Map.Entry> entry : - cookies.entrySet()) { - String key = entry.getKey(); - // ignore all entries that don't have "Cookie" - // or "Cookie2" as keys - if (!"Cookie".equalsIgnoreCase(key) && - !"Cookie2".equalsIgnoreCase(key)) { - continue; - } - List l = entry.getValue(); - if (l != null && !l.isEmpty()) { - requests.add(key, String.join("; ", l)); - } - } - } - } - if (userCookies != null) { - int k; - if ((k = requests.getKey("Cookie")) != -1) - requests.set("Cookie", requests.getValue(k) + ";" + userCookies); - else - requests.set("Cookie", userCookies); - } - if (userCookies2 != null) { - int k; - if ((k = requests.getKey("Cookie2")) != -1) - requests.set("Cookie2", requests.getValue(k) + ";" + userCookies2); - else - requests.set("Cookie2", userCookies2); - } - - } // end of getting cookies - } - - @Override - public InputStream getInputStream() throws IOException { - lock(); - try { - connecting = true; - return getInputStream0(); - } finally { - unlock(); - } - } - - @SuppressWarnings("empty-statement") - private InputStream getInputStream0() throws IOException { - - assert isLockHeldByCurrentThread(); - if (!doInput) { - throw new ProtocolException("Cannot read from URLConnection" - + " if doInput=false (call setDoInput(true))"); - } - - if (rememberedException != null) { - if (rememberedException instanceof RuntimeException) - throw new RuntimeException(rememberedException); - else { - throw getChainedException((IOException)rememberedException); - } - } - - if (inputStream != null) { - return inputStream; - } - - if (streaming() ) { - if (strOutputStream == null) { - getOutputStream(); - } - /* make sure stream is closed */ - strOutputStream.close (); - if (!strOutputStream.writtenOK()) { - throw new IOException ("Incomplete output stream"); - } - } - - int redirects = 0; - int respCode = 0; - long cl = -1; - AuthenticationInfo serverAuthentication = null; - AuthenticationInfo proxyAuthentication = null; - AuthenticationHeader srvHdr = null; - - /** - * Failed Negotiate - * - * In some cases, the Negotiate auth is supported for the - * remote host but the negotiate process still fails (For - * example, if the web page is located on a backend server - * and delegation is needed but fails). The authentication - * process will start again, and we need to detect this - * kind of failure and do proper fallback (say, to NTLM). - * - * In order to achieve this, the inNegotiate flag is set - * when the first negotiate challenge is met (and reset - * if authentication is finished). If a fresh new negotiate - * challenge (no parameter) is found while inNegotiate is - * set, we know there's a failed auth attempt recently. - * Here we'll ignore the header line so that fallback - * can be practiced. - * - * inNegotiateProxy is for proxy authentication. - */ - boolean inNegotiate = false; - boolean inNegotiateProxy = false; - - // If the user has set either of these headers then do not remove them - isUserServerAuth = requests.getKey("Authorization") != -1; - isUserProxyAuth = requests.getKey("Proxy-Authorization") != -1; - try { - do { - if (!checkReuseConnection()) - connect(); - - if (cachedInputStream != null) { - return cachedInputStream; - } - - // we may need to remove proxy-authorization - Proxy p = http.getHttpProxy(); - // if we're not using a proxy or if the proxy to be used is not - // the same as the originally set one, then remove it - if (p == null || (lastProxy != null && !lastProxy.equals(p))) { - requests.remove("Proxy-Authorization"); - lastProxy = null; - } - /* REMIND: This exists to fix the HttpsURLConnection subclass. - * Hotjava needs to run on JDK1.1FCS. Do proper fix once a - * proper solution for SSL can be found. - */ - ps = (PrintStream)http.getOutputStream(); - - if (!streaming()) { - writeRequests(); - } - http.parseHTTP(responses, this); - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine(responses.toString()); - } - - boolean b1 = responses.filterNTLMResponses("WWW-Authenticate"); - boolean b2 = responses.filterNTLMResponses("Proxy-Authenticate"); - if (b1 || b2) { - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine(">>>> Headers are filtered"); - logger.fine(responses.toString()); - } - } - - inputStream = http.getInputStream(); - - respCode = getResponseCode(); - if (respCode == -1) { - disconnectInternal(); - throw new IOException ("Invalid Http response"); - } - if (respCode == HTTP_PROXY_AUTH && tunnelState() != TunnelState.TUNNELING) { - if (streaming()) { - disconnectInternal(); - throw new HttpRetryException ( - RETRY_MSG1, HTTP_PROXY_AUTH); - } - - // Read comments labeled "Failed Negotiate" for details. - boolean dontUseNegotiate = false; - Iterator iter = responses.multiValueIterator("Proxy-Authenticate"); - while (iter.hasNext()) { - String value = iter.next().trim(); - if (value.equalsIgnoreCase("Negotiate") || - value.equalsIgnoreCase("Kerberos")) { - if (!inNegotiateProxy) { - inNegotiateProxy = true; - } else { - dontUseNegotiate = true; - doingNTLMp2ndStage = false; - proxyAuthentication = null; - } - break; - } - } - - // changes: add a 3rd parameter to the constructor of - // AuthenticationHeader, so that NegotiateAuthentication. - // isSupported can be tested. - // The other 2 appearances of "new AuthenticationHeader" is - // altered in similar ways. - - AuthenticationHeader authhdr = new AuthenticationHeader ( - "Proxy-Authenticate", - responses, - getHttpCallerInfo(url, - http.getProxyHostUsed(), - http.getProxyPortUsed(), - authenticator), - dontUseNegotiate, - disabledProxyingSchemes - ); - - if (!doingNTLMp2ndStage) { - proxyAuthentication = - resetProxyAuthentication(proxyAuthentication, authhdr); - if (proxyAuthentication != null) { - redirects++; - disconnectInternal(); - continue; - } - } else { - /* in this case, only one header field will be present */ - String raw = responses.findValue ("Proxy-Authenticate"); - reset (); - try { - proxyAuthentication.setHeaders(this, - authhdr.headerParser(), raw); - } catch (IOException ex) { - disconnectInternal(); - if (Exceptions.enhancedNonSocketExceptions()) { - throw new IOException ("Authentication failure", ex); - } else { - throw new IOException ("Authentication failure"); - } - } - if (serverAuthentication != null && srvHdr != null) { - try { - serverAuthentication.setHeaders(this, - srvHdr.headerParser(), raw); - } catch (IOException ex) { - disconnectInternal(); - if (Exceptions.enhancedNonSocketExceptions()) { - throw new IOException ("Authentication failure", ex); - } else { - throw new IOException ("Authentication failure"); - } - } - } - authObj = null; - doingNTLMp2ndStage = false; - continue; - } - } else { - inNegotiateProxy = false; - doingNTLMp2ndStage = false; - if (!isUserProxyAuth) - requests.remove("Proxy-Authorization"); - } - - // cache proxy authentication info - if (proxyAuthentication != null) { - // cache auth info on success, domain header not relevant. - proxyAuthentication.addToCache(authCache); - } - - if (respCode == HTTP_UNAUTHORIZED) { - if (streaming()) { - disconnectInternal(); - throw new HttpRetryException ( - RETRY_MSG2, HTTP_UNAUTHORIZED); - } - - // Read comments labeled "Failed Negotiate" for details. - boolean dontUseNegotiate = false; - Iterator iter = responses.multiValueIterator("WWW-Authenticate"); - while (iter.hasNext()) { - String value = iter.next().trim(); - if (value.equalsIgnoreCase("Negotiate") || - value.equalsIgnoreCase("Kerberos")) { - if (!inNegotiate) { - inNegotiate = true; - } else { - dontUseNegotiate = true; - doingNTLM2ndStage = false; - serverAuthentication = null; - } - break; - } - } - - srvHdr = new AuthenticationHeader ( - "WWW-Authenticate", responses, - getHttpCallerInfo(url, authenticator), - dontUseNegotiate - ); - - String raw = srvHdr.raw(); - if (!doingNTLM2ndStage) { - if ((serverAuthentication != null)&& - serverAuthentication.getAuthScheme() != NTLM) { - if (serverAuthentication.isAuthorizationStale (raw)) { - /* we can retry with the current credentials */ - disconnectWeb(); - redirects++; - requests.set(serverAuthentication.getHeaderName(), - serverAuthentication.getHeaderValue(url, method)); - currentServerCredentials = serverAuthentication; - setCookieHeader(); - continue; - } else { - serverAuthentication.removeFromCache(authCache); - } - } - serverAuthentication = getServerAuthentication(srvHdr); - currentServerCredentials = serverAuthentication; - - if (serverAuthentication != null) { - disconnectWeb(); - redirects++; // don't let things loop ad nauseum - setCookieHeader(); - continue; - } - } else { - reset (); - /* header not used for ntlm */ - try { - serverAuthentication.setHeaders(this, null, raw); - } catch (IOException ex) { - disconnectWeb(); - if (Exceptions.enhancedNonSocketExceptions()) { - throw new IOException ("Authentication failure", ex); - } else { - throw new IOException ("Authentication failure"); - } - } - doingNTLM2ndStage = false; - authObj = null; - setCookieHeader(); - continue; - } - } - // cache server authentication info - if (serverAuthentication != null) { - // cache auth info on success - if (!(serverAuthentication instanceof DigestAuthentication) || - (domain == null)) { - if (serverAuthentication instanceof BasicAuthentication) { - // check if the path is shorter than the existing entry - String npath = AuthenticationInfo.reducePath (url.getPath()); - String opath = serverAuthentication.path; - if (!opath.startsWith (npath) || npath.length() >= opath.length()) { - /* npath is longer, there must be a common root */ - npath = BasicAuthentication.getRootPath (opath, npath); - } - // remove the entry and create a new one - BasicAuthentication a = - (BasicAuthentication) serverAuthentication.clone(); - serverAuthentication.removeFromCache(authCache); - a.path = npath; - serverAuthentication = a; - } - serverAuthentication.addToCache(authCache); - } else { - // what we cache is based on the domain list in the request - DigestAuthentication srv = (DigestAuthentication) - serverAuthentication; - StringTokenizer tok = new StringTokenizer (domain," "); - String realm = srv.realm; - PasswordAuthentication pw = srv.pw; - digestparams = srv.params; - while (tok.hasMoreTokens()) { - String path = tok.nextToken(); - try { - /* path could be an abs_path or a complete URI */ - URL u = newURL (url, path); - DigestAuthentication d = new DigestAuthentication ( - false, u, realm, "Digest", pw, - digestparams); - d.addToCache (authCache); - } catch (Exception e) {} - } - } - } - - // some flags should be reset to its initialized form so that - // even after a redirect the necessary checks can still be - // performed. - inNegotiate = false; - inNegotiateProxy = false; - - //serverAuthentication = null; - doingNTLMp2ndStage = false; - doingNTLM2ndStage = false; - if (!isUserServerAuth) - requests.remove("Authorization"); - if (!isUserProxyAuth) - requests.remove("Proxy-Authorization"); - - if (respCode == HTTP_OK) { - checkResponseCredentials (false); - } else { - needToCheck = false; - } - - // a flag need to clean - needToCheck = true; - - if (followRedirect()) { - /* if we should follow a redirect, then the followRedirects() - * method will disconnect() and re-connect us to the new - * location - */ - redirects++; - - // redirecting HTTP response may have set cookie, so - // need to re-generate request header - setCookieHeader(); - - continue; - } - - final String contentLengthVal = responses.findValue("content-length"); - if (contentLengthVal != null) { - try { - cl = Long.parseLong(contentLengthVal); - } catch (NumberFormatException nfe) { } - } - - if (method.equals("HEAD") || cl == 0 || - respCode == HTTP_NOT_MODIFIED || - respCode == HTTP_NO_CONTENT) { - noResponseBody(); - } - - if (respCode == 200 || respCode == 203 || respCode == 206 || - respCode == 300 || respCode == 301 || respCode == 410) { - if (cacheHandler != null && getUseCaches()) { - // give cache a chance to save response in cache - URI uri = ParseUtil.toURI(url); - if (uri != null) { - URLConnection uconn = this; - if ("https".equalsIgnoreCase(uri.getScheme())) { - try { - // use reflection to get to the public - // HttpsURLConnection instance saved in - // DelegateHttpsURLConnection - uconn = (URLConnection)this.getClass().getField("httpsURLConnection").get(this); - } catch (IllegalAccessException | - NoSuchFieldException e) { - // ignored; use 'this' - } - } - CacheRequest cacheRequest = - cacheHandler.put(uri, uconn); - if (cacheRequest != null && http != null) { - http.setCacheRequest(cacheRequest); - inputStream = new HttpInputStream(inputStream, cacheRequest); - } - } - } - } - - if (!(inputStream instanceof HttpInputStream)) { - inputStream = new HttpInputStream(inputStream); - } - - if (respCode >= 400) { - if (respCode == 404 || respCode == 410) { - throw new FileNotFoundException(url.toString()); - } else { - throw new java.io.IOException("Server returned HTTP" + - " response code: " + respCode + " for URL: " + - url.toString()); - } - } - poster = null; - strOutputStream = null; - return inputStream; - } while (redirects < maxRedirects); - - throw new ProtocolException("Server redirected too many times (" + - redirects + ")"); - } catch (RuntimeException e) { - disconnectInternal(); - rememberedException = e; - throw e; - } catch (IOException e) { - rememberedException = e; - - // buffer the error stream if bytes < 4k - // and it can be buffered within 1 second - String te = responses.findValue("Transfer-Encoding"); - if (http != null && http.isKeepingAlive() && enableESBuffer && - (cl > 0 || (te != null && te.equalsIgnoreCase("chunked")))) { - errorStream = ErrorStream.getErrorStream(inputStream, cl, http); - } - throw e; - } finally { - if (proxyAuthKey != null) { - AuthenticationInfo.endAuthRequest(proxyAuthKey); - } - if (serverAuthKey != null) { - AuthenticationInfo.endAuthRequest(serverAuthKey); - } - if (proxyAuthentication != null) { - proxyAuthentication.disposeContext(); - } - if (serverAuthentication != null) { - serverAuthentication.disposeContext(); - } - } - } - - /** - * This method is called when a response with no response - * body is received, and arrange for the http client to - * be returned to the pool (or released) immediately when - * possible. - * @apiNote Used by {@link sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection} - * to preserve the TLS information after receiving an empty body. - * @implSpec - * Subclasses that override this method should call the super class - * implementation. - */ - protected void noResponseBody() { - http.finished(); - http = null; - inputStream = new EmptyInputStream(); - connected = false; - } - - /* - * Creates a chained exception that has the same type as - * original exception and with the same message. Right now, - * there is no convenient APIs for doing so. - */ - private IOException getChainedException(final IOException rememberedException) { - try { - final Object[] args = { rememberedException.getMessage() }; - IOException chainedException = rememberedException.getClass() - .getConstructor(new Class[] { String.class }) - .newInstance(args); - chainedException.initCause(rememberedException); - return chainedException; - } catch (Exception ignored) { - return rememberedException; - } - } - - @Override - public InputStream getErrorStream() { - if (connected && responseCode >= 400) { - // Client Error 4xx and Server Error 5xx - if (errorStream != null) { - return errorStream; - } else if (inputStream != null) { - return inputStream; - } - } - return null; - } - - /** - * set or reset proxy authentication info in request headers - * after receiving a 407 error. In the case of NTLM however, - * receiving a 407 is normal and we just skip the stale check - * because ntlm does not support this feature. - */ - private AuthenticationInfo - resetProxyAuthentication(AuthenticationInfo proxyAuthentication, AuthenticationHeader auth) throws IOException { - - // Only called from getInputStream0 and doTunneling0 - assert isLockHeldByCurrentThread(); - - if ((proxyAuthentication != null )&& - proxyAuthentication.getAuthScheme() != NTLM) { - String raw = auth.raw(); - if (proxyAuthentication.isAuthorizationStale (raw)) { - /* we can retry with the current credentials */ - String value; - if (proxyAuthentication instanceof DigestAuthentication) { - DigestAuthentication digestProxy = (DigestAuthentication) - proxyAuthentication; - if (tunnelState() == TunnelState.SETUP) { - value = digestProxy.getHeaderValue(connectRequestURI(url), HTTP_CONNECT); - } else { - value = digestProxy.getHeaderValue(getRequestURI(), method); - } - } else { - value = proxyAuthentication.getHeaderValue(url, method); - } - requests.set(proxyAuthentication.getHeaderName(), value); - currentProxyCredentials = proxyAuthentication; - return proxyAuthentication; - } else { - proxyAuthentication.removeFromCache(authCache); - } - } - proxyAuthentication = getHttpProxyAuthentication(auth); - currentProxyCredentials = proxyAuthentication; - return proxyAuthentication; - } - - /** - * Returns the tunnel state. - * - * @return the state - */ - TunnelState tunnelState() { - return tunnelState; - } - - /** - * Set the tunneling status. - * - * @param tunnelState the state - */ - public void setTunnelState(TunnelState tunnelState) { - this.tunnelState = tunnelState; - } - - /** - * establish a tunnel through proxy server - */ - public void doTunneling() throws IOException { - lock(); - try { - doTunneling0(); - } finally{ - unlock(); - } - } - - private void doTunneling0() throws IOException { - int retryTunnel = 0; - String statusLine = ""; - int respCode = 0; - AuthenticationInfo proxyAuthentication = null; - String proxyHost = null; - int proxyPort = -1; - - assert isLockHeldByCurrentThread(); - - // save current requests so that they can be restored after tunnel is setup. - MessageHeader savedRequests = requests; - requests = new MessageHeader(); - - // Read comments labeled "Failed Negotiate" for details. - boolean inNegotiateProxy = false; - - try { - /* Actively setting up a tunnel */ - setTunnelState(TunnelState.SETUP); - - do { - if (!checkReuseConnection()) { - proxiedConnect(url, proxyHost, proxyPort, false); - } - // send the "CONNECT" request to establish a tunnel - // through proxy server - sendCONNECTRequest(); - responses.reset(); - - http.parseHTTP(responses, this); - - /* Log the response to the CONNECT */ - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine(responses.toString()); - } - - if (responses.filterNTLMResponses("Proxy-Authenticate")) { - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine(">>>> Headers are filtered"); - logger.fine(responses.toString()); - } - } - - statusLine = responses.getValue(0); - respCode = parseConnectResponseCode(statusLine); - if (respCode == -1) { - // a respCode of -1, due to a invalid status line, - // will (rightly) result in an IOException being thrown - // later in this code. here we merely log the invalid status line. - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine("invalid status line: \"" + statusLine + "\""); - } - } - if (respCode == HTTP_PROXY_AUTH) { - // Read comments labeled "Failed Negotiate" for details. - boolean dontUseNegotiate = false; - Iterator iter = responses.multiValueIterator("Proxy-Authenticate"); - while (iter.hasNext()) { - String value = iter.next().trim(); - if (value.equalsIgnoreCase("Negotiate") || - value.equalsIgnoreCase("Kerberos")) { - if (!inNegotiateProxy) { - inNegotiateProxy = true; - } else { - dontUseNegotiate = true; - doingNTLMp2ndStage = false; - proxyAuthentication = null; - } - break; - } - } - - AuthenticationHeader authhdr = new AuthenticationHeader( - "Proxy-Authenticate", - responses, - getHttpCallerInfo(url, - http.getProxyHostUsed(), - http.getProxyPortUsed(), - authenticator), - dontUseNegotiate, - disabledTunnelingSchemes - ); - if (!doingNTLMp2ndStage) { - proxyAuthentication = - resetProxyAuthentication(proxyAuthentication, authhdr); - if (proxyAuthentication != null) { - proxyHost = http.getProxyHostUsed(); - proxyPort = http.getProxyPortUsed(); - disconnectInternal(); - retryTunnel++; - continue; - } - } else { - String raw = responses.findValue ("Proxy-Authenticate"); - reset (); - try { - proxyAuthentication.setHeaders(this, - authhdr.headerParser(), raw); - } catch (IOException ex) { - disconnectInternal(); - if (Exceptions.enhancedNonSocketExceptions()) { - throw new IOException ("Authentication failure", ex); - } else { - throw new IOException ("Authentication failure"); - } - } - authObj = null; - doingNTLMp2ndStage = false; - continue; - } - } - // cache proxy authentication info - if (proxyAuthentication != null) { - // cache auth info on success, domain header not relevant. - proxyAuthentication.addToCache(authCache); - } - - if (respCode == HTTP_OK) { - setTunnelState(TunnelState.TUNNELING); - savedRequests.remove("Proxy-Authorization"); - break; - } - // we don't know how to deal with other response code - // so disconnect and report error - disconnectInternal(); - setTunnelState(TunnelState.NONE); - break; - } while (retryTunnel < maxRedirects); - - if (retryTunnel >= maxRedirects || (respCode != HTTP_OK)) { - if (respCode != HTTP_PROXY_AUTH) { - // remove all but authenticate responses - responses.reset(); - } - throw new IOException("Unable to tunnel through proxy."+ - " Proxy returns \"" + - statusLine + "\""); - } - } finally { - if (proxyAuthKey != null) { - AuthenticationInfo.endAuthRequest(proxyAuthKey); - } - if (proxyAuthentication != null) { - proxyAuthentication.disposeContext(); - } - } - - // restore original request headers - requests = savedRequests; - - // reset responses - responses.reset(); - } - - // parses the status line, that was returned for a CONNECT request, and returns - // the response code from that line. returns -1 if the response code could not be - // parsed. - private static int parseConnectResponseCode(final String statusLine) { - final int invalidStatusLine = -1; - if (statusLine == null || statusLine.isBlank()) { - return invalidStatusLine; - } - // - // status-line = HTTP-version SP status-code SP [ reason-phrase ] - // SP = space character - // - final StringTokenizer st = new StringTokenizer(statusLine, " "); - if (!st.hasMoreTokens()) { - return invalidStatusLine; - } - st.nextToken(); // the HTTP version part (ex: HTTP/1.1) - if (!st.hasMoreTokens()) { - return invalidStatusLine; - } - final String v = st.nextToken().trim(); // status code - try { - return Integer.parseInt(v); - } catch (NumberFormatException nfe) { - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine("invalid response code: " + v); - } - } - return invalidStatusLine; - } - - /** - * Overridden in https to also include the server certificate - */ - protected HttpCallerInfo getHttpCallerInfo(URL url, String proxy, int port, - Authenticator authenticator) { - return new HttpCallerInfo(url, proxy, port, authenticator); - } - - /** - * Overridden in https to also include the server certificate - */ - protected HttpCallerInfo getHttpCallerInfo(URL url, Authenticator authenticator) { - return new HttpCallerInfo(url, authenticator); - } - - static String connectRequestURI(URL url) { - String host = url.getHost(); - int port = url.getPort(); - port = port != -1 ? port : url.getDefaultPort(); - - return host + ":" + port; - } - - /** - * send a CONNECT request for establishing a tunnel to proxy server - */ - private void sendCONNECTRequest() throws IOException { - int port = url.getPort(); - - requests.set(0, HTTP_CONNECT + " " + connectRequestURI(url) - + " " + httpVersion, null); - requests.setIfNotSet("User-Agent", userAgent); - - String host = url.getHost(); - if (port != -1 && port != url.getDefaultPort()) { - host += ":" + String.valueOf(port); - } - requests.setIfNotSet("Host", host); - - // Not really necessary for a tunnel, but can't hurt - requests.setIfNotSet("Accept", acceptString); - - if (http.getHttpKeepAliveSet()) { - requests.setIfNotSet("Proxy-Connection", "keep-alive"); - } - - setPreemptiveProxyAuthentication(requests); - - /* Log the CONNECT request */ - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine(requests.toString()); - } - - http.writeRequests(requests, null); - } - - /** - * Sets pre-emptive proxy authentication in header - */ - private void setPreemptiveProxyAuthentication(MessageHeader requests) throws IOException { - AuthenticationInfo pauth - = AuthenticationInfo.getProxyAuth(http.getProxyHostUsed(), - http.getProxyPortUsed(), - authCache); - if (pauth != null && pauth.supportsPreemptiveAuthorization()) { - String value; - if (pauth instanceof DigestAuthentication) { - DigestAuthentication digestProxy = (DigestAuthentication) pauth; - if (tunnelState() == TunnelState.SETUP) { - value = digestProxy - .getHeaderValue(connectRequestURI(url), HTTP_CONNECT); - } else { - value = digestProxy.getHeaderValue(getRequestURI(), method); - } - } else { - value = pauth.getHeaderValue(url, method); - } - - // Sets "Proxy-authorization" - requests.set(pauth.getHeaderName(), value); - currentProxyCredentials = pauth; - } - } - - /** - * Gets the authentication for an HTTP proxy, and applies it to - * the connection. - */ - private AuthenticationInfo getHttpProxyAuthentication(AuthenticationHeader authhdr) - throws IOException { - - assert isLockHeldByCurrentThread(); - - /* get authorization from authenticator */ - AuthenticationInfo ret = null; - String raw = authhdr.raw(); - String host = http.getProxyHostUsed(); - int port = http.getProxyPortUsed(); - if (host != null && authhdr.isPresent()) { - HeaderParser p = authhdr.headerParser(); - String realm = p.findValue("realm"); - String charset = p.findValue("charset"); - boolean isUTF8 = (charset != null && charset.equalsIgnoreCase("UTF-8")); - String scheme = authhdr.scheme(); - AuthScheme authScheme = UNKNOWN; - if ("basic".equalsIgnoreCase(scheme)) { - authScheme = BASIC; - } else if ("digest".equalsIgnoreCase(scheme)) { - authScheme = DIGEST; - } else if ("ntlm".equalsIgnoreCase(scheme)) { - authScheme = NTLM; - doingNTLMp2ndStage = true; - } else if ("Kerberos".equalsIgnoreCase(scheme)) { - authScheme = KERBEROS; - doingNTLMp2ndStage = true; - } else if ("Negotiate".equalsIgnoreCase(scheme)) { - authScheme = NEGOTIATE; - doingNTLMp2ndStage = true; - } - - if (realm == null) - realm = ""; - proxyAuthKey = AuthenticationInfo.getProxyAuthKey(host, port, realm, authScheme); - ret = AuthenticationInfo.getProxyAuth(proxyAuthKey, authCache); - if (ret == null) { - ret = switch (authScheme) { - case BASIC -> { - InetAddress addr = null; - try { - addr = InetAddress.getByName(host); - } catch (UnknownHostException uhe) { - // Ignore the exception. The Authenticator instance will - // be passed a null InetAddress when requesting a password from the - // Authenticator. - } - final PasswordAuthentication a = requestPassword(authenticator, - host, addr, port, "http", - realm, scheme, url, RequestorType.PROXY); - if (a != null) { - yield new BasicAuthentication(true, host, port, realm, a, isUTF8); - } - yield null; - } - case DIGEST -> { - final PasswordAuthentication a = requestPassword(authenticator, - host, null, port, url.getProtocol(), - realm, scheme, url, RequestorType.PROXY); - if (a != null) { - DigestAuthentication.Parameters params = - new DigestAuthentication.Parameters(); - yield new DigestAuthentication(true, host, port, realm, - scheme, a, params); - } - yield null; - } - case NTLM -> { - if (!NTLMAuthenticationProxy.supported) { - yield null; - } - /* tryTransparentNTLMProxy will always be true the first - * time around, but verify that the platform supports it - * otherwise don't try. */ - if (tryTransparentNTLMProxy) { - tryTransparentNTLMProxy = - NTLMAuthenticationProxy.supportsTransparentAuth; - /* If the platform supports transparent authentication - * then normally it's ok to do transparent auth to a proxy - * because we generally trust proxies (chosen by the user) - * But not in the case of 305 response where the server - * chose it. */ - if (tryTransparentNTLMProxy && useProxyResponseCode) { - tryTransparentNTLMProxy = false; - } - - } - PasswordAuthentication a = null; - if (tryTransparentNTLMProxy) { - logger.finest("Trying Transparent NTLM authentication"); - } else { - a = requestPassword( - authenticator, - host, null, port, url.getProtocol(), - "", scheme, url, RequestorType.PROXY); - validateNTLMCredentials(a); - } - /* If we are not trying transparent authentication then - * we need to have a PasswordAuthentication instance. For - * transparent authentication (Windows only) the username - * and password will be picked up from the current logged - * on users credentials. - */ - AuthenticationInfo authInfo = null; - if (tryTransparentNTLMProxy || - (!tryTransparentNTLMProxy && a != null)) { - authInfo = NTLMAuthenticationProxy.proxy.create(true, host, port, a); - } - - /* set to false so that we do not try again */ - tryTransparentNTLMProxy = false; - yield authInfo; - } - case NEGOTIATE -> - new NegotiateAuthentication(new HttpCallerInfo(authhdr.getHttpCallerInfo(), "Negotiate")); - case KERBEROS -> - new NegotiateAuthentication(new HttpCallerInfo(authhdr.getHttpCallerInfo(), "Kerberos")); - case UNKNOWN -> { - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - logger.finest("Unknown/Unsupported authentication scheme: " + scheme); - } - yield null; - } - }; - } - if (ret != null) { - try { - ret.setHeaders(this, p, raw); - } catch (IOException e) { - ret.disposeContext(); - ret = null; - } - } - } - if (logger.isLoggable(PlatformLogger.Level.FINER)) { - logger.finer("Proxy Authentication for " + authhdr.toString() +" returned " + (ret != null ? ret.toString() : "null")); - } - return ret; - } - - /** - * Gets the authentication for an HTTP server, and applies it to - * the connection. - * @param authhdr the AuthenticationHeader which tells what auth scheme is - * preferred. - */ - @SuppressWarnings("fallthrough") - private AuthenticationInfo getServerAuthentication(AuthenticationHeader authhdr) - throws IOException { - - // Only called from getInputStream0 - assert isLockHeldByCurrentThread(); - - /* get authorization from authenticator */ - AuthenticationInfo ret = null; - String raw = authhdr.raw(); - /* When we get an NTLM auth from cache, don't set any special headers */ - if (authhdr.isPresent()) { - HeaderParser p = authhdr.headerParser(); - String realm = p.findValue("realm"); - String scheme = authhdr.scheme(); - String charset = p.findValue("charset"); - boolean isUTF8 = (charset != null && charset.equalsIgnoreCase("UTF-8")); - AuthScheme authScheme = UNKNOWN; - if ("basic".equalsIgnoreCase(scheme)) { - authScheme = BASIC; - } else if ("digest".equalsIgnoreCase(scheme)) { - authScheme = DIGEST; - } else if ("ntlm".equalsIgnoreCase(scheme)) { - authScheme = NTLM; - doingNTLM2ndStage = true; - } else if ("Kerberos".equalsIgnoreCase(scheme)) { - authScheme = KERBEROS; - doingNTLM2ndStage = true; - } else if ("Negotiate".equalsIgnoreCase(scheme)) { - authScheme = NEGOTIATE; - doingNTLM2ndStage = true; - } - - domain = p.findValue ("domain"); - if (realm == null) - realm = ""; - serverAuthKey = AuthenticationInfo.getServerAuthKey(url, realm, authScheme); - ret = AuthenticationInfo.getServerAuth(serverAuthKey, authCache); - InetAddress addr = null; - if (ret == null) { - try { - addr = InetAddress.getByName(url.getHost()); - } catch (java.net.UnknownHostException ignored) { - // User will have addr = null - } - } - // replacing -1 with default port for a protocol - int port = url.getPort(); - if (port == -1) { - port = url.getDefaultPort(); - } - if (ret == null) { - switch(authScheme) { - case KERBEROS: - ret = new NegotiateAuthentication(new HttpCallerInfo(authhdr.getHttpCallerInfo(), "Kerberos")); - break; - case NEGOTIATE: - ret = new NegotiateAuthentication(new HttpCallerInfo(authhdr.getHttpCallerInfo(), "Negotiate")); - break; - case BASIC: - PasswordAuthentication a = - requestPassword( - authenticator, - url.getHost(), addr, port, url.getProtocol(), - realm, scheme, url, RequestorType.SERVER); - if (a != null) { - ret = new BasicAuthentication(false, url, realm, a, isUTF8); - } - break; - case DIGEST: - a = requestPassword( - authenticator, - url.getHost(), addr, port, url.getProtocol(), - realm, scheme, url, RequestorType.SERVER); - if (a != null) { - digestparams = new DigestAuthentication.Parameters(); - ret = new DigestAuthentication(false, url, realm, scheme, - a, digestparams); - } - break; - case NTLM: - if (NTLMAuthenticationProxy.supported) { - URL url1; - try { - url1 = newURL (url, "/"); /* truncate the path */ - } catch (Exception e) { - url1 = url; - } - - /* tryTransparentNTLMServer will always be true the first - * time around, but verify that the platform supports it - * otherwise don't try. */ - if (tryTransparentNTLMServer) { - tryTransparentNTLMServer = - NTLMAuthenticationProxy.supportsTransparentAuth; - /* If the platform supports transparent authentication - * then check if we are in a secure environment - * whether, or not, we should try transparent authentication.*/ - if (tryTransparentNTLMServer) { - tryTransparentNTLMServer = - NTLMAuthenticationProxy.isTrustedSite(url); - } - } - a = null; - if (tryTransparentNTLMServer) { - logger.finest("Trying Transparent NTLM authentication"); - } else { - a = requestPassword( - authenticator, - url.getHost(), addr, port, url.getProtocol(), - "", scheme, url, RequestorType.SERVER); - validateNTLMCredentials(a); - } - - /* If we are not trying transparent authentication then - * we need to have a PasswordAuthentication instance. For - * transparent authentication (Windows only) the username - * and password will be picked up from the current logged - * on users credentials. - */ - if (tryTransparentNTLMServer || - (!tryTransparentNTLMServer && a != null)) { - ret = NTLMAuthenticationProxy.proxy.create(false, url1, a); - } - - /* set to false so that we do not try again */ - tryTransparentNTLMServer = false; - } - break; - case UNKNOWN: - if (logger.isLoggable(PlatformLogger.Level.FINEST)) { - logger.finest("Unknown/Unsupported authentication scheme: " + scheme); - } - /*fall through*/ - default: - throw new AssertionError("should not reach here"); - } - } - if (ret != null ) { - try { - ret.setHeaders(this, p, raw); - } catch (IOException e) { - ret.disposeContext(); - ret = null; - } - } - } - if (logger.isLoggable(PlatformLogger.Level.FINER)) { - logger.finer("Server Authentication for " + authhdr.toString() +" returned " + (ret != null ? ret.toString() : "null")); - } - return ret; - } - - /* inclose will be true if called from close(), in which case we - * force the call to check because this is the last chance to do so. - * If not in close(), then the authentication info could arrive in a trailer - * field, which we have not read yet. - */ - private void checkResponseCredentials (boolean inClose) throws IOException { - try { - if (!needToCheck) - return; - if ((validateProxy && currentProxyCredentials != null) && - (currentProxyCredentials instanceof DigestAuthentication)) { - String raw = responses.findValue ("Proxy-Authentication-Info"); - if (inClose || (raw != null)) { - DigestAuthentication da = (DigestAuthentication) - currentProxyCredentials; - da.checkResponse (raw, method, getRequestURI()); - currentProxyCredentials.disposeContext(); - currentProxyCredentials = null; - } - } - if ((validateServer && currentServerCredentials != null) && - (currentServerCredentials instanceof DigestAuthentication)) { - String raw = responses.findValue ("Authentication-Info"); - if (inClose || (raw != null)) { - DigestAuthentication da = (DigestAuthentication) - currentServerCredentials; - da.checkResponse (raw, method, url); - currentServerCredentials.disposeContext(); - currentServerCredentials = null; - } - } - if ((currentServerCredentials==null) && (currentProxyCredentials == null)) { - needToCheck = false; - } - } catch (IOException e) { - disconnectInternal(); - connected = false; - throw e; - } - } - - /* The request URI used in the request line for this request. - * Also, needed for digest authentication - */ - - String requestURI = null; - - String getRequestURI() throws IOException { - if (requestURI == null) { - requestURI = http.getURLFile(); - } - return requestURI; - } - - /* Tells us whether to follow a redirect. If so, it - * closes the connection (break any keep-alive) and - * resets the url, re-connects, and resets the request - * property. - */ - private boolean followRedirect() throws IOException { - if (!getInstanceFollowRedirects()) { - return false; - } - - final int stat = getResponseCode(); - if (stat < 300 || stat > 307 || stat == 306 - || stat == HTTP_NOT_MODIFIED) { - return false; - } - final String loc = getHeaderField("Location"); - if (loc == null) { - /* this should be present - if not, we have no choice - * but to go forward w/ the response we got - */ - return false; - } - - URL locUrl; - try { - locUrl = newURL(loc); - if (!url.getProtocol().equalsIgnoreCase(locUrl.getProtocol())) { - return false; - } - - } catch (MalformedURLException mue) { - // treat loc as a relative URI to conform to popular browsers - locUrl = newURL(url, loc); - } - return followRedirect0(loc, stat, locUrl); - } - - /* Tells us whether to follow a redirect. If so, it - * closes the connection (break any keep-alive) and - * resets the url, re-connects, and resets the request - * property. - */ - private boolean followRedirect0(String loc, int stat, URL locUrl) - throws IOException - { - assert isLockHeldByCurrentThread(); - - lastProxy = http.getHttpProxy(); - disconnectInternal(); - if (streaming()) { - throw new HttpRetryException (RETRY_MSG3, stat, loc); - } - if (logger.isLoggable(PlatformLogger.Level.FINE)) { - logger.fine("Redirected from " + url + " to " + locUrl); - } - - // clear out old response headers!!!! - responses = new MessageHeader(maxHeaderSize); - if (stat == HTTP_USE_PROXY) { - /* This means we must re-request the resource through the - * proxy denoted in the "Location:" field of the response. - * Judging by the spec, the string in the Location header - * _should_ denote a URL - let's hope for "http://my.proxy.org" - * Make a new HttpClient to the proxy, using HttpClient's - * Instance-specific proxy fields, but note we're still fetching - * the same URL. - */ - String proxyHost = locUrl.getHost(); - int proxyPort = locUrl.getPort(); - - setProxiedClient (url, proxyHost, proxyPort); - requests.set(0, method + " " + getRequestURI()+" " + - httpVersion, null); - connected = true; - // need to remember this in case NTLM proxy authentication gets - // used. We can't use transparent authentication when user - // doesn't know about proxy. - useProxyResponseCode = true; - } else { - final URL prevURL = url; - - // maintain previous headers, just change the name - // of the file we're getting - url = locUrl; - requestURI = null; // force it to be recalculated - if (method.equals("POST") && !Boolean.getBoolean("http.strictPostRedirect") && (stat!=307)) { - /* The HTTP/1.1 spec says that a redirect from a POST - * *should not* be immediately turned into a GET, and - * that some HTTP/1.0 clients incorrectly did this. - * Correct behavior redirects a POST to another POST. - * Unfortunately, since most browsers have this incorrect - * behavior, the web works this way now. Typical usage - * seems to be: - * POST a login code or passwd to a web page. - * after validation, the server redirects to another - * (welcome) page - * The second request is (erroneously) expected to be GET - * - * We will do the incorrect thing (POST-->GET) by default. - * We will provide the capability to do the "right" thing - * (POST-->POST) by a system property, "http.strictPostRedirect=true" - */ - - requests = new MessageHeader(); - setRequests = false; - super.setRequestMethod("GET"); // avoid the connecting check - poster = null; - if (!checkReuseConnection()) - connect(); - - if (!sameDestination(prevURL, url)) { - // Ensures pre-redirect user-set cookie will not be reset. - // CookieHandler, if any, will be queried to determine - // cookies for redirected URL, if any. - userCookies = null; - userCookies2 = null; - } - } else { - if (!checkReuseConnection()) - connect(); - /* Even after a connect() call, http variable still can be - * null, if a ResponseCache has been installed and it returns - * a non-null CacheResponse instance. So check nullity before using it. - * - * And further, if http is null, there's no need to do anything - * about request headers because successive http session will use - * cachedInputStream/cachedHeaders anyway, which is returned by - * CacheResponse. - */ - if (http != null) { - requests.set(0, method + " " + getRequestURI()+" " + - httpVersion, null); - int port = url.getPort(); - String host = stripIPv6ZoneId(url.getHost()); - if (port != -1 && port != url.getDefaultPort()) { - host += ":" + String.valueOf(port); - } - requests.set("Host", host); - } - - if (!sameDestination(prevURL, url)) { - // Redirecting to a different destination will drop any - // security-sensitive headers, regardless of whether - // they are user-set or not. CookieHandler, if any, will be - // queried to determine cookies for redirected URL, if any. - userCookies = null; - userCookies2 = null; - requests.remove("Cookie"); - requests.remove("Cookie2"); - requests.remove("Authorization"); - - // check for preemptive authorization - AuthenticationInfo sauth = - AuthenticationInfo.getServerAuth(url, authCache); - if (sauth != null && sauth.supportsPreemptiveAuthorization() ) { - // Sets "Authorization" - requests.setIfNotSet(sauth.getHeaderName(), sauth.getHeaderValue(url,method)); - currentServerCredentials = sauth; - } - } - } - } - return true; - } - - /* Returns true iff the given URLs have the same host and effective port. */ - private static boolean sameDestination(URL firstURL, URL secondURL) { - assert firstURL.getProtocol().equalsIgnoreCase(secondURL.getProtocol()): - "protocols not equal: " + firstURL + " - " + secondURL; - - if (!firstURL.getHost().equalsIgnoreCase(secondURL.getHost())) - return false; - - int firstPort = firstURL.getPort(); - if (firstPort == -1) - firstPort = firstURL.getDefaultPort(); - int secondPort = secondURL.getPort(); - if (secondPort == -1) - secondPort = secondURL.getDefaultPort(); - if (firstPort != secondPort) - return false; - - return true; - } - - /* dummy byte buffer for reading off socket prior to closing */ - byte[] cdata = new byte [128]; - - /** - * Reset (without disconnecting the TCP conn) in order to do another transaction with this instance - */ - private void reset() throws IOException { - http.reuse = true; - /* must save before calling close */ - reuseClient = http; - InputStream is = http.getInputStream(); - if (!method.equals("HEAD") || tunnelState == TunnelState.SETUP) { - try { - /* we want to read the rest of the response without using the - * hurry mechanism, because that would close the connection - * if everything is not available immediately - */ - if ((is instanceof ChunkedInputStream) || - (is instanceof MeteredStream)) { - /* reading until eof will not block */ - while (is.read (cdata) > 0) {} - } else { - /* raw stream, which will block on read, so only read - * the expected number of bytes, probably 0 - */ - long cl = 0; - int n = 0; - String cls = responses.findValue ("Content-Length"); - if (cls != null) { - try { - cl = Long.parseLong (cls); - } catch (NumberFormatException e) { - cl = 0; - } - } - for (long i=0; i cookies = access.parse(value); - for (HttpCookie cookie : cookies) { - // skip HttpOnly cookies - if (!cookie.isHttpOnly()) - retValue.add(access.header(cookie)); - } - return retValue.toString(); - } - - return value; - } - - // Cache the filtered response headers so that they don't need - // to be generated for every getHeaderFields() call. - private Map> filteredHeaders; // null - - private Map> getFilteredHeaderFields() { - if (filteredHeaders != null) - return filteredHeaders; - - Map> headers, tmpMap = new HashMap<>(); - - if (cachedHeaders != null) - headers = cachedHeaders.getHeaders(); - else - headers = responses.getHeaders(); - - for (Map.Entry> e: headers.entrySet()) { - String key = e.getKey(); - List values = e.getValue(), filteredVals = new ArrayList<>(); - for (String value : values) { - String fVal = filterHeaderField(key, value); - if (fVal != null) - filteredVals.add(fVal); - } - if (!filteredVals.isEmpty()) - tmpMap.put(key, Collections.unmodifiableList(filteredVals)); - } - - return filteredHeaders = Collections.unmodifiableMap(tmpMap); - } - - /** - * Gets a header field by name. Returns null if not known. - * @param name the name of the header field - */ - @Override - public String getHeaderField(String name) { - try { - getInputStream(); - } catch (IOException e) {} - - if (cachedHeaders != null) { - return filterHeaderField(name, cachedHeaders.findValue(name)); - } - - return filterHeaderField(name, responses.findValue(name)); - } - - /** - * Returns an unmodifiable Map of the header fields. - * The Map keys are Strings that represent the - * response-header field names. Each Map value is an - * unmodifiable List of Strings that represents - * the corresponding field values. - * - * @return a Map of header fields - * @since 1.4 - */ - @Override - public Map> getHeaderFields() { - try { - getInputStream(); - } catch (IOException e) {} - - return getFilteredHeaderFields(); - } - - /** - * Gets a header field by index. Returns null if not known. - * @param n the index of the header field - */ - @Override - public String getHeaderField(int n) { - try { - getInputStream(); - } catch (IOException e) {} - - if (cachedHeaders != null) { - return filterHeaderField(cachedHeaders.getKey(n), - cachedHeaders.getValue(n)); - } - return filterHeaderField(responses.getKey(n), responses.getValue(n)); - } - - /** - * Gets a header field by index. Returns null if not known. - * @param n the index of the header field - */ - @Override - public String getHeaderFieldKey(int n) { - try { - getInputStream(); - } catch (IOException e) {} - - if (cachedHeaders != null) { - return cachedHeaders.getKey(n); - } - - return responses.getKey(n); - } - - /** - * Sets request property. If a property with the key already - * exists, overwrite its value with the new value. - * @param value the value to be set - */ - @Override - public void setRequestProperty(String key, String value) { - lock(); - try { - if (connected || connecting) - throw new IllegalStateException("Already connected"); - if (key == null) - throw new NullPointerException("key is null"); - - if (isExternalMessageHeaderAllowed(key, value)) { - requests.set(key, value); - if (!key.equalsIgnoreCase("Content-Type")) { - userHeaders.set(key, value); - } - } - } finally { - unlock(); - } - } - - MessageHeader getUserSetHeaders() { - return userHeaders; - } - - /** - * Adds a general request property specified by a - * key-value pair. This method will not overwrite - * existing values associated with the same key. - * - * @param key the keyword by which the request is known - * (e.g., "accept"). - * @param value the value associated with it. - * @see #getRequestProperty(java.lang.String) - * @since 1.4 - */ - @Override - public void addRequestProperty(String key, String value) { - lock(); - try { - if (connected || connecting) - throw new IllegalStateException("Already connected"); - if (key == null) - throw new NullPointerException("key is null"); - - if (isExternalMessageHeaderAllowed(key, value)) { - requests.add(key, value); - if (!key.equalsIgnoreCase("Content-Type")) { - userHeaders.add(key, value); - } - } - } finally { - unlock(); - } - } - - // - // Set a property for authentication. This can safely disregard - // the connected test. - // - public void setAuthenticationProperty(String key, String value) { - // Only called by the implementation of AuthenticationInfo::setHeaders(...) - // in AuthenticationInfo subclasses, which is only called from - // methods from HttpURLConnection protected by the connectionLock. - assert isLockHeldByCurrentThread(); - - checkMessageHeader(key, value); - requests.set(key, value); - } - - @Override - public String getRequestProperty (String key) { - lock(); - try { - if (key == null) { - return null; - } - - // don't return headers containing security sensitive information - for (int i = 0; i < EXCLUDE_HEADERS.length; i++) { - if (key.equalsIgnoreCase(EXCLUDE_HEADERS[i])) { - return null; - } - } - if (!setUserCookies) { - if (key.equalsIgnoreCase("Cookie")) { - return userCookies; - } - if (key.equalsIgnoreCase("Cookie2")) { - return userCookies2; - } - } - return requests.findValue(key); - } finally { - unlock(); - } - } - - /** - * Returns an unmodifiable Map of general request - * properties for this connection. The Map keys - * are Strings that represent the request-header - * field names. Each Map value is a unmodifiable List - * of Strings that represents the corresponding - * field values. - * - * @return a Map of the general request properties for this connection. - * @throws IllegalStateException if already connected - * @since 1.4 - */ - @Override - public Map> getRequestProperties() { - lock(); - try { - if (connected) - throw new IllegalStateException("Already connected"); - - // exclude headers containing security-sensitive info - if (setUserCookies) { - return requests.getHeaders(EXCLUDE_HEADERS); - } - /* - * The cookies in the requests message headers may have - * been modified. Use the saved user cookies instead. - */ - Map> userCookiesMap = null; - if (userCookies != null || userCookies2 != null) { - userCookiesMap = new HashMap<>(); - if (userCookies != null) { - userCookiesMap.put("Cookie", Arrays.asList(userCookies)); - } - if (userCookies2 != null) { - userCookiesMap.put("Cookie2", Arrays.asList(userCookies2)); - } - } - return requests.filterAndAddHeaders(EXCLUDE_HEADERS2, userCookiesMap); - } finally { - unlock(); - } - } - - @Override - public void setConnectTimeout(int timeout) { - if (timeout < 0) - throw new IllegalArgumentException("timeouts can't be negative"); - connectTimeout = timeout; - } - - - /** - * Returns setting for connect timeout. - *

- * 0 return implies that the option is disabled - * (i.e., timeout of infinity). - * - * @return an int that indicates the connect timeout - * value in milliseconds - * @see java.net.URLConnection#setConnectTimeout(int) - * @see java.net.URLConnection#connect() - * @since 1.5 - */ - @Override - public int getConnectTimeout() { - return (connectTimeout < 0 ? 0 : connectTimeout); - } - - /** - * Sets the read timeout to a specified timeout, in - * milliseconds. A non-zero value specifies the timeout when - * reading from Input stream when a connection is established to a - * resource. If the timeout expires before there is data available - * for read, a java.net.SocketTimeoutException is raised. A - * timeout of zero is interpreted as an infinite timeout. - * - *

Some non-standard implementation of this method ignores the - * specified timeout. To see the read timeout set, please call - * getReadTimeout(). - * - * @param timeout an int that specifies the timeout - * value to be used in milliseconds - * @throws IllegalArgumentException if the timeout parameter is negative - * - * @see java.net.URLConnection#getReadTimeout() - * @see java.io.InputStream#read() - * @since 1.5 - */ - @Override - public void setReadTimeout(int timeout) { - if (timeout < 0) - throw new IllegalArgumentException("timeouts can't be negative"); - readTimeout = timeout; - } - - /** - * Returns setting for read timeout. 0 return implies that the - * option is disabled (i.e., timeout of infinity). - * - * @return an int that indicates the read timeout - * value in milliseconds - * - * @see java.net.URLConnection#setReadTimeout(int) - * @see java.io.InputStream#read() - * @since 1.5 - */ - @Override - public int getReadTimeout() { - return readTimeout < 0 ? 0 : readTimeout; - } - - public CookieHandler getCookieHandler() { - return cookieHandler; - } - - String getMethod() { - return method; - } - - private MessageHeader mapToMessageHeader(Map> map) { - MessageHeader headers = new MessageHeader(); - if (map == null || map.isEmpty()) { - return headers; - } - for (Map.Entry> entry : map.entrySet()) { - String key = entry.getKey(); - List values = entry.getValue(); - for (String value : values) { - if (key == null) { - headers.prepend(key, value); - } else { - headers.add(key, value); - } - } - } - return headers; - } - - /** - * Returns the given host, without the IPv6 Zone Id, if present. - * (e.g. [fe80::a00:27ff:aaaa:aaaa%eth0] -> [fe80::a00:27ff:aaaa:aaaa]) - * - * @param host host address (not null, not empty) - * @return host address without Zone Id - */ - static String stripIPv6ZoneId(String host) { - if (host.charAt(0) != '[') { // not an IPv6-literal - return host; - } - int i = host.lastIndexOf('%'); - if (i == -1) { // doesn't contain zone_id - return host; - } - return host.substring(0, i) + "]"; - } - - /* The purpose of this wrapper is just to capture the close() call - * so we can check authentication information that may have - * arrived in a Trailer field - */ - class HttpInputStream extends FilterInputStream { - private CacheRequest cacheRequest; - private OutputStream outputStream; - private boolean marked = false; - private int inCache = 0; - private int markCount = 0; - private boolean closed; // false - - public HttpInputStream (InputStream is) { - super (is); - this.cacheRequest = null; - this.outputStream = null; - } - - public HttpInputStream (InputStream is, CacheRequest cacheRequest) { - super (is); - this.cacheRequest = cacheRequest; - try { - this.outputStream = cacheRequest.getBody(); - } catch (IOException ioex) { - this.cacheRequest.abort(); - this.cacheRequest = null; - this.outputStream = null; - } - } - - /** - * Marks the current position in this input stream. A subsequent - * call to the reset method repositions this stream at - * the last marked position so that subsequent reads re-read the same - * bytes. - *

- * The readlimit argument tells this input stream to - * allow that many bytes to be read before the mark position gets - * invalidated. - *

- * This method simply performs in.mark(readlimit). - * - * @param readlimit the maximum limit of bytes that can be read before - * the mark position becomes invalid. - * @see java.io.FilterInputStream#in - * @see java.io.FilterInputStream#reset() - */ - // safe to use synchronized here: super method is synchronized too - // and involves no blocking operation; only mark & reset are - // synchronized in the super class hierarchy. - @Override - public synchronized void mark(int readlimit) { - super.mark(readlimit); - if (cacheRequest != null) { - marked = true; - markCount = 0; - } - } - - /** - * Repositions this stream to the position at the time the - * mark method was last called on this input stream. - *

- * This method - * simply performs in.reset(). - *

- * Stream marks are intended to be used in - * situations where you need to read ahead a little to see what's in - * the stream. Often this is most easily done by invoking some - * general parser. If the stream is of the type handled by the - * parse, it just chugs along happily. If the stream is not of - * that type, the parser should toss an exception when it fails. - * If this happens within readlimit bytes, it allows the outer - * code to reset the stream and try another parser. - * - * @exception IOException if the stream has not been marked or if the - * mark has been invalidated. - * @see java.io.FilterInputStream#in - * @see java.io.FilterInputStream#mark(int) - */ - // safe to use synchronized here: super method is synchronized too - // and involves no blocking operation; only mark & reset are - // synchronized in the super class hierarchy. - @Override - public synchronized void reset() throws IOException { - super.reset(); - if (cacheRequest != null) { - marked = false; - inCache += markCount; - } - } - - private void ensureOpen() throws IOException { - if (closed) - throw new IOException("stream is closed"); - } - - @Override - public int read() throws IOException { - ensureOpen(); - try { - byte[] b = new byte[1]; - int ret = read(b); - return (ret == -1? ret : (b[0] & 0x00FF)); - } catch (IOException ioex) { - if (cacheRequest != null) { - cacheRequest.abort(); - } - throw ioex; - } - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - ensureOpen(); - try { - int newLen = super.read(b, off, len); - int nWrite; - // write to cache - if (inCache > 0) { - if (inCache >= newLen) { - inCache -= newLen; - nWrite = 0; - } else { - nWrite = newLen - inCache; - inCache = 0; - } - } else { - nWrite = newLen; - } - if (nWrite > 0 && outputStream != null) - outputStream.write(b, off + (newLen-nWrite), nWrite); - if (marked) { - markCount += newLen; - } - return newLen; - } catch (IOException ioex) { - if (cacheRequest != null) { - cacheRequest.abort(); - } - throw ioex; - } - } - - /* skip() calls read() in order to ensure that entire response gets - * cached. same implementation as InputStream.skip */ - - private byte[] skipBuffer; - private static final int SKIP_BUFFER_SIZE = 8096; - - @Override - public long skip (long n) throws IOException { - ensureOpen(); - long remaining = n; - int nr; - if (skipBuffer == null) - skipBuffer = new byte[SKIP_BUFFER_SIZE]; - - byte[] localSkipBuffer = skipBuffer; - - if (n <= 0) { - return 0; - } - - while (remaining > 0) { - nr = read(localSkipBuffer, 0, - (int) Math.min(SKIP_BUFFER_SIZE, remaining)); - if (nr < 0) { - break; - } - remaining -= nr; - } - - return n - remaining; - } - - @Override - public void close () throws IOException { - if (closed) - return; - - try { - if (outputStream != null) { - if (read() != -1) { - cacheRequest.abort(); - } else { - outputStream.close(); - } - } - super.close (); - } catch (IOException ioex) { - if (cacheRequest != null) { - cacheRequest.abort(); - } - throw ioex; - } finally { - closed = true; - HttpURLConnection.this.http = null; - checkResponseCredentials (true); - } - } - } - - class StreamingOutputStream extends FilterOutputStream { - - long expected; - long written; - boolean closed; - boolean error; - IOException errorExcp; - - /** - * expectedLength == -1 if the stream is chunked - * expectedLength > 0 if the stream is fixed content-length - * In the 2nd case, we make sure the expected number - * of bytes are actually written - */ - StreamingOutputStream (OutputStream os, long expectedLength) { - super (os); - expected = expectedLength; - written = 0L; - closed = false; - error = false; - } - - @Override - public void write (int b) throws IOException { - checkError(); - written ++; - if (expected != -1L && written > expected) { - throw new IOException ("too many bytes written"); - } - out.write (b); - } - - @Override - public void write (byte[] b) throws IOException { - write (b, 0, b.length); - } - - @Override - public void write (byte[] b, int off, int len) throws IOException { - checkError(); - written += len; - if (expected != -1L && written > expected) { - out.close (); - throw new IOException ("too many bytes written"); - } - out.write (b, off, len); - } - - void checkError () throws IOException { - if (closed) { - throw new IOException ("Stream is closed"); - } - if (error) { - throw errorExcp; - } - if (out instanceof PrintStream) { - if (((PrintStream) out).checkError()) { - throw new IOException("Error writing request body to server"); - } - } else if (out instanceof ChunkedOutputStream) { - if (((ChunkedOutputStream) out).checkError()) { - throw new IOException("Error writing request body to server"); - } - } - } - - /* this is called to check that all the bytes - * that were supposed to be written were written - * and that the stream is now closed(). - */ - boolean writtenOK () { - return closed && ! error; - } - - @Override - public void close () throws IOException { - if (closed) { - return; - } - closed = true; - if (expected != -1L) { - /* not chunked */ - if (written != expected) { - error = true; - errorExcp = new IOException ("insufficient data written"); - out.close (); - throw errorExcp; - } - super.flush(); /* can't close the socket */ - } else { - /* chunked */ - super.close (); /* force final chunk to be written */ - /* trailing \r\n */ - OutputStream o = http.getOutputStream(); - o.write ('\r'); - o.write ('\n'); - o.flush(); - } - } - } - - - static class ErrorStream extends InputStream { - ByteBuffer buffer; - InputStream is; - - private ErrorStream(ByteBuffer buf) { - buffer = buf; - is = null; - } - - private ErrorStream(ByteBuffer buf, InputStream is) { - buffer = buf; - this.is = is; - } - - // when this method is called, it's either the case that cl > 0, or - // if chunk-encoded, cl = -1; in other words, cl can't be 0 - public static InputStream getErrorStream(InputStream is, long cl, HttpClient http) { - - // cl can't be 0; this following is here for extra precaution - if (cl == 0) { - return null; - } - - try { - // set SO_TIMEOUT to 1/5th of the total timeout - // remember the old timeout value so that we can restore it - int oldTimeout = http.getReadTimeout(); - http.setReadTimeout(timeout4ESBuffer/5); - - long expected = 0; - boolean isChunked = false; - // the chunked case - if (cl < 0) { - expected = bufSize4ES; - isChunked = true; - } else { - expected = cl; - } - if (expected <= bufSize4ES) { - int exp = (int) expected; - byte[] buffer = new byte[exp]; - int count = 0, time = 0, len = 0; - do { - try { - len = is.read(buffer, count, - buffer.length - count); - if (len < 0) { - if (isChunked) { - // chunked ended - // if chunked ended prematurely, - // an IOException would be thrown - break; - } - // the server sends less than cl bytes of data - throw new IOException("the server closes"+ - " before sending "+cl+ - " bytes of data"); - } - count += len; - } catch (SocketTimeoutException ex) { - time += timeout4ESBuffer/5; - } - } while (count < exp && time < timeout4ESBuffer); - - // reset SO_TIMEOUT to old value - http.setReadTimeout(oldTimeout); - - // if count < cl at this point, we will not try to reuse - // the connection - if (count == 0) { - // since we haven't read anything, - // we will return the underlying - // inputstream back to the application - return null; - } else if ((count == expected && !(isChunked)) || (isChunked && len <0)) { - // put the connection into keep-alive cache - // the inputstream will try to do the right thing - is.close(); - return new ErrorStream(ByteBuffer.wrap(buffer, 0, count)); - } else { - // we read part of the response body - return new ErrorStream( - ByteBuffer.wrap(buffer, 0, count), is); - } - } - return null; - } catch (IOException ioex) { - // ioex.printStackTrace(); - return null; - } - } - - @Override - public int available() throws IOException { - if (is == null) { - return buffer.remaining(); - } else { - return buffer.remaining()+is.available(); - } - } - - public int read() throws IOException { - byte[] b = new byte[1]; - int ret = read(b); - return (ret == -1? ret : (b[0] & 0x00FF)); - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int rem = buffer.remaining(); - if (rem > 0) { - int ret = rem < len? rem : len; - buffer.get(b, off, ret); - return ret; - } else { - if (is == null) { - return -1; - } else { - return is.read(b, off, len); - } - } - } - - @Override - public void close() throws IOException { - buffer = null; - if (is != null) { - is.close(); - } - } - } - - @SuppressWarnings("deprecation") - private static URL newURL(String spec) throws MalformedURLException { - return new URL(spec); - } - - @SuppressWarnings("deprecation") - private static URL newURL(URL context, String spec) throws MalformedURLException { - return new URL(context, spec); - } - - // ensure there are no null characters in username or password - private static void validateNTLMCredentials(PasswordAuthentication pw) - throws IOException { - - if (pw == null) { - return; - } - char[] password = pw.getPassword(); - if (password != null) { - for (int i=0; i