+ * This exists because the two platforms are not obviously comparable. iOS encodes with {@code Codable}, + * Android encodes with {@code EventOutputFormatter} in java-sdk-internal, and a JIT with escape + * analysis behaves nothing like ahead-of-time compiled Swift. The iOS finding -- that the cost is in + * the serializer's per-value machinery rather than in walking and redacting the context -- is a claim + * about {@code Codable}, and nothing about it transfers here by assumption. + *
+ * Instrumented copy of the JVM {@code EventPersistenceBenchmark}: same corpus, same loops, running + * on ART on a device rather than on a desktop JVM. Skipped unless the instrumentation argument + * {@code LD_EVENT_BENCH=1} is set, because nothing here asserts anything a regression would trip: + *
+ * ./gradlew :launchdarkly-android-client-sdk:connectedDebugAndroidTest \\ + * -Pandroid.testInstrumentationRunnerArguments.LD_EVENT_BENCH=1 \\ + * -Pandroid.testInstrumentationRunnerArguments.class=com.launchdarkly.sdk.android.EventPersistenceBenchmark + *+ * Results are written to {@code event-bench.txt} in the target app's files directory (pulled after + * the run) as well as to logcat, because Gradle's instrumented-test logger does not reliably keep + * {@code System.out}. + */ +public class EventPersistenceBenchmark { + private static final String FLAG_KEY = "benchmark-flag"; + private static final int FLAG_VERSION = 7; + private static final int VARIATION = 1; + + /** + * Fixed rather than {@code System.currentTimeMillis()}, so that repeated runs encode identical + * bytes and the clock call is not charged to the encoder. The event-construction row below covers + * what the real path pays around it. + */ + private static final long CREATION_DATE = 1_700_000_000_000L; + + /** + * One already serialized event, byte for byte what the iOS benchmark stages, so the store figures on the + * two platforms are measuring the same amount of work reaching the same kind of file. + */ + private static final String SERIALIZED_EVENT = + "{\"kind\":\"feature\",\"key\":\"benchmark-flag\",\"value\":true,\"default\":false," + + "\"variation\":1,\"version\":7,\"creationDate\":1740000000000}"; + + /** How many times each timed loop is repeated; the fastest round is reported. */ + private static final int ROUNDS = 5; + + /** + * The sizes the commit-point figures are swept over. The same three are used by the iOS benchmark, so + * the two platforms' tables line up row for row. + */ + private static final int[] BULK_SIZES = {100, 1_000, 10_000}; + + /** + * Consumes results so that the JIT cannot prove the work is unused and delete it. Never read for + * its value. + */ + private static long blackhole; + + /** + * Holds an allocation so that escape analysis cannot scalar-replace it. Assigning to a static field + * is what makes the object genuinely exist, which is the difference between measuring an allocation + * and measuring nothing at all. + */ + private static Object escaped; + + @Rule + public TemporaryFolder eventsDirectory = new TemporaryFolder( + InstrumentationRegistry.getInstrumentation().getTargetContext().getCacheDir()); + + private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); + + private static PrintWriter logFile; + + @BeforeClass + public static void openLog() throws IOException { + File file = new File( + InstrumentationRegistry.getInstrumentation().getTargetContext().getFilesDir(), + "event-bench.txt"); + logFile = new PrintWriter(new FileWriter(file, true), true); + logFile.println("device=" + android.os.Build.MODEL + + " manufacturer=" + android.os.Build.MANUFACTURER + + " device=" + android.os.Build.DEVICE + + " abi=" + android.os.Build.SUPPORTED_ABIS[0] + + " sdk=" + android.os.Build.VERSION.SDK_INT + + " release=" + android.os.Build.VERSION.RELEASE + + " hardware=" + android.os.Build.HARDWARE); + } + + @AfterClass + public static void closeLog() { + if (logFile != null) { + logFile.close(); + } + } + + @Before + public void requireBenchmarking() { + assumeTrue("Set instrumentation argument LD_EVENT_BENCH=1 to measure event recording", + "1".equals(InstrumentationRegistry.getArguments().getString("LD_EVENT_BENCH"))); + } + + /** + * M1 and M2 for Android: where the time goes when one feature event is serialized, across the + * context shapes and privacy settings that change how much work redaction has to do. + *
+ * The shape of the result is the interesting part. If the cost is dominated by walking and
+ * redacting the context, it should grow with the number of attributes and grow again when private
+ * attributes force the filtering path. If it is dominated by the serializer's fixed per-event
+ * machinery, it should be far flatter than the attribute counts suggest.
+ */
+ @Test
+ public void encodingCostByContextShape() {
+ List
+ * The three rows are the three paths through {@code recordEvaluationEvent} and {@code recordCustomEvent}:
+ * an evaluation of an untracked flag is a counter increment; an evaluation of a tracked flag also
+ * serializes and stages a full event; and a {@code track} is a commit point, which additionally turns
+ * the counters into summary events and writes everything to the log.
+ *
+ * The sequence is reproduced here rather than driven through the processor itself, which would need
+ * an HTTP server, a scheduler and a {@code ClientContext} whose costs would swamp what is being
+ * measured. It is kept deliberately close to the original; if the processor changes, this should too.
+ */
+ @Test
+ public void recordingCostPerCall() throws IOException {
+ final OutboundEventBuffer buffer = makeBuffer(noRedaction());
+ final EventStore store = EventStore.create(eventsDirectory.getRoot(), "benchmark-key",
+ Integer.MAX_VALUE, true, logger);
+ final LDContext context = makeContext(ContextShape.STUB, "benchmark-key");
+
+ try {
+ List
+ * {@code track} and {@code identify} commit on the caller's thread by design -- that is the durable
+ * barrier -- so this is the one figure here an application can observe as a stall. The mean is the number
+ * that gets quoted; the tail is the number a user notices.
+ *
+ * The counterpart iOS test runs this against a SQLite store as well, where the tail is the point: a
+ * database in WAL mode folds its write-ahead log back into the file every thousand pages, on whichever
+ * thread happens to be committing, so one unlucky caller pays for all of them. An append has no such
+ * thing, and these rows are what that claim is measured against.
+ *
+ * Two rows per size. {@code track} commits where it stands, so its tail is the write. The other row
+ * encodes and stages without committing, and the store never writes on its own, so its tail is the
+ * encode and the copy; the gap between the two is what the write itself costs.
+ *
+ * p99.9 only separates from the worst at ten thousand samples. Below that the index falls on the
+ * last sample and the two columns are the same number by construction. They are left side by side so the
+ * table lines up across the sweep, and so the size at which they part is visible rather than asserted.
+ */
+ @Test
+ public void commitPointOutliers() throws IOException {
+ final byte[] serialized = SERIALIZED_EVENT.getBytes(StandardCharsets.UTF_8);
+ final OutboundEventBuffer buffer = makeBuffer(noRedaction());
+ final LDContext context = makeContext(ContextShape.STUB, "benchmark-key");
+
+ // A store of its own for each size and each row, so that a log inherited from an earlier measurement
+ // does not charge the next one for the file it had already grown.
+ for (int calls : BULK_SIZES) {
+ final EventStore committing = EventStore.create(eventsDirectory.newFolder(), "benchmark-key",
+ Integer.MAX_VALUE, true, logger);
+ final EventStore staging = EventStore.create(eventsDirectory.newFolder(), "benchmark-key",
+ Integer.MAX_VALUE, true, logger);
+ try {
+ List
+ * Android differs from iOS in a way that matters. {@code LDContext} is an immutable object passed by
+ * reference, and nothing on the event path copies it, so an application that holds one context hands
+ * the same instance to every event and {@code equals} can settle on reference identity. iOS had to
+ * reach that fast path by not rebuilding a value type. The row that decides the question is the other
+ * one: an application that rebuilds its context between evaluations produces an equal object sharing
+ * nothing, and that comparison is the price of a cache hit in the worst case.
+ */
+ @Test
+ public void contextComparisonCost() {
+ List
+ * Counting and serializing are reported separately, because only the second is deferred -- the
+ * counter increment is already paid on every evaluation.
+ */
+ @Test
+ public void summarySerializationCost() {
+ final LDContext context = makeContext(ContextShape.STUB, "benchmark-key");
+ List
+ * Warmed first, because a JVM measured cold reports the interpreter rather than the compiled code.
+ * Then run several times with the fastest round reported: a garbage collection or a descheduled
+ * thread can only ever make a round slower, so the minimum is the closest estimate of the work
+ * itself. That is also why these numbers should not be read as what a busy device would see.
+ */
+ private double measure(int iterations, IntConsumer body) {
+ int warmup = Math.max(10_000, iterations / 4);
+ for (int i = 0; i < warmup; i++) {
+ body.accept(i);
+ }
+
+ double best = Double.MAX_VALUE;
+ for (int round = 0; round < ROUNDS; round++) {
+ long start = System.nanoTime();
+ for (int i = 0; i < iterations; i++) {
+ body.accept(i);
+ }
+ long elapsed = System.nanoTime() - start;
+ best = Math.min(best, (double) elapsed / iterations);
+ }
+ return best;
+ }
+
+ /** How long one call took, across many of them, when the tail is the point. */
+ private static final class Distribution {
+ final double median;
+ final double mean;
+ final double p999;
+ final double worst;
+
+ Distribution(double median, double mean, double p999, double worst) {
+ this.median = median;
+ this.mean = mean;
+ this.p999 = p999;
+ this.worst = worst;
+ }
+ }
+
+ /**
+ * Times every call rather than the enclosing loop, so the tail survives to be reported.
+ *
+ * One round, and the median rather than the fastest. Every other figure here reports the cheapest of
+ * several rounds, on the reasoning that noise can only make a round slower -- but here an outlier is the
+ * measurement, and that aggregation would throw away the answer.
+ */
+ private Distribution measureDistribution(int iterations, IntConsumer body) {
+ int warmup = Math.max(10_000, iterations / 4);
+ for (int i = 0; i < warmup; i++) {
+ body.accept(i);
+ }
+
+ long[] samples = new long[iterations];
+ for (int i = 0; i < iterations; i++) {
+ long start = System.nanoTime();
+ body.accept(i);
+ samples[i] = System.nanoTime() - start;
+ }
+
+ long total = 0;
+ for (long sample : samples) {
+ total += sample;
+ }
+ Arrays.sort(samples);
+ return new Distribution(samples[iterations / 2],
+ (double) total / iterations,
+ samples[Math.min(iterations - 1, (int) ((long) iterations * 999 / 1000))],
+ samples[iterations - 1]);
+ }
+
+ private static final class DistributionMeasurement {
+ final String name;
+ final Distribution distribution;
+
+ DistributionMeasurement(String name, Distribution distribution) {
+ this.name = name;
+ this.distribution = distribution;
+ }
+ }
+
+ private static final class Measurement {
+ final String name;
+ final double nanosPerOp;
+
+ Measurement(String name, double nanosPerOp) {
+ this.name = name;
+ this.nanosPerOp = nanosPerOp;
+ }
+ }
+
+ private static void report(String title, List
+ * This exists for the sake of one header. java-sdk-internal's sender invents a payload ID per call,
+ * which is the right thing when the events being sent only ever existed in memory: the sole reason to
+ * send the same events twice is its own immediate retry, and it holds the ID for that. Once events are
+ * kept on disk, a delivery can also be repeated by a later run of the application, after a process died
+ * without learning whether the service had received the batch. Those two attempts are the same delivery
+ * and have to say so, or the events are counted twice - and counting them twice is worse than losing
+ * them, because an inflated exposure count silently biases an experiment rather than visibly shrinking
+ * it.
+ *
+ * The ID belongs to the batch, which is why it is the batch's identity in {@link EventStore} rather than
+ * something generated at send time.
+ */
+final class AnalyticsEventSender implements Closeable {
+ private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8");
+ private static final String EVENT_SCHEMA_HEADER = "X-LaunchDarkly-Event-Schema";
+ private static final String EVENT_SCHEMA_VERSION = "4";
+ private static final String PAYLOAD_ID_HEADER = "X-LaunchDarkly-Payload-ID";
+
+ private final OkHttpClient client;
+ private final boolean ownsClient;
+ private final Headers baseHeaders;
+ private final LDLogger logger;
+
+ AnalyticsEventSender(HttpProperties httpProperties, LDLogger logger) {
+ this.logger = logger;
+ if (httpProperties.getSharedHttpClient() == null) {
+ this.client = httpProperties.toHttpClientBuilder().build();
+ this.ownsClient = true;
+ } else {
+ this.client = httpProperties.getSharedHttpClient();
+ this.ownsClient = false;
+ }
+ this.baseHeaders = httpProperties.toHeadersBuilder().build();
+ }
+
+ /** What one post came to. */
+ static final class Result {
+ /** Whether the service took the events. */
+ final boolean success;
+ /**
+ * For a failure, whether it is unlikely to pass soon: a 4xx other than a refusal of the payload
+ * ({@link LDUtil#isEventPayloadRefused}), 408 and 429, or a TLS or certificate failure. This only
+ * decides how long the next post waits.
+ */
+ final boolean unexpected;
+ /** The HTTP status, or 0 where no response arrived. */
+ final int status;
+ /** The wait a failed response asked for with {@code Retry-After}, or -1 where it asked for none. */
+ final long retryAfterMillis;
+ /** The service's clock, from an accepted response's {@code Date}, or null. */
+ final Date serverTime;
+
+ Result(boolean success, boolean unexpected, int status, long retryAfterMillis, Date serverTime) {
+ this.success = success;
+ this.unexpected = unexpected;
+ this.status = status;
+ this.retryAfterMillis = retryAfterMillis;
+ this.serverTime = serverTime;
+ }
+ }
+
+ /**
+ * Posts a batch once. The retry is the caller's, so that the wait before it does not hold a thread.
+ *
+ * @param payload the JSON request body, sent from the array it is in rather than copied out of it
+ * @param eventCount how many events the body holds, for logging
+ * @param payloadId identifies this delivery, the same on every attempt at it
+ * @param eventsBaseUri the events service
+ * @return whether the events arrived and, where they did not, how long the next attempt should wait
+ */
+ Result sendBatch(EventLogFormat.Payload payload, int eventCount, String payloadId, URI eventsBaseUri) {
+ URI uri = HttpHelpers.concatenateUriPath(eventsBaseUri,
+ StandardEndpoints.ANALYTICS_EVENTS_REQUEST_PATH);
+ Request request = new Request.Builder()
+ .url(uri.toString())
+ .headers(baseHeaders)
+ .addHeader("Content-Type", "application/json")
+ .addHeader(EVENT_SCHEMA_HEADER, EVENT_SCHEMA_VERSION)
+ .addHeader(PAYLOAD_ID_HEADER, payloadId)
+ .post(RequestBody.create(payload.bytes, JSON, 0, payload.length))
+ .build();
+
+ try (Response response = client.newCall(request).execute()) {
+ if (response.isSuccessful()) {
+ logger.debug("Posted {} event(s)", eventCount);
+ return new Result(true, false, response.code(), -1, parseDate(response));
+ }
+ logger.warn("Error posting events: HTTP status {}", response.code());
+ // A refusal of the payload waits like a 400: the caller drops the batch after a second in a row,
+ // and the batches behind it are not to be held back an extended wait meanwhile.
+ boolean unexpected = LDUtil.isHttpErrorUnexpected(response.code())
+ && !LDUtil.isEventPayloadRefused(response.code());
+ return new Result(false, unexpected, response.code(), retryAfterMillis(response), null);
+ } catch (IOException e) {
+ logger.warn("Error posting events: {}", LogValues.exceptionSummary(e));
+ // A certificate that will not validate is a misconfiguration, which waits like a refusal.
+ boolean unexpected = e instanceof SSLHandshakeException || e instanceof SSLPeerUnverifiedException;
+ return new Result(false, unexpected, 0, -1, null);
+ }
+ }
+
+ /** {@code Retry-After} as seconds or as an HTTP date, or -1 where it is absent or unreadable. */
+ private static long retryAfterMillis(Response response) {
+ String value = response.header("Retry-After");
+ if (value == null) {
+ return -1;
+ }
+ try {
+ long seconds = Long.parseLong(value.trim());
+ return seconds < 0 ? -1 : TimeUnit.SECONDS.toMillis(seconds);
+ } catch (NumberFormatException e) {
+ // An HTTP date, then.
+ }
+ Date date;
+ try {
+ date = response.headers().getDate("Retry-After");
+ } catch (RuntimeException e) {
+ return -1;
+ }
+ return date == null ? -1 : Math.max(0, date.getTime() - System.currentTimeMillis());
+ }
+
+ /**
+ * The service's own clock, which the SDK trusts over the device's when deciding whether a debug
+ * window has closed.
+ */
+ private Date parseDate(Response response) {
+ try {
+ return response.headers().getDate("Date");
+ } catch (RuntimeException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public void close() {
+ if (!ownsClient) {
+ return;
+ }
+ client.dispatcher().executorService().shutdown();
+ client.connectionPool().evictAll();
+ }
+}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java
index 357648917..ac6e90aee 100644
--- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java
@@ -169,6 +169,11 @@ public File getCacheDir() {
return application.getCacheDir();
}
+ @Override
+ public File getNoBackupFilesDir() {
+ return application.getNoBackupFilesDir();
+ }
+
@Override
public void close() {
connectivityChangeListeners.clear();
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/BatchLocks.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/BatchLocks.java
new file mode 100644
index 000000000..a29deed2a
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/BatchLocks.java
@@ -0,0 +1,200 @@
+package com.launchdarkly.sdk.android;
+
+import android.annotation.SuppressLint;
+import android.os.ParcelFileDescriptor;
+
+import androidx.annotation.VisibleForTesting;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.StandardOpenOption;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * The locks that tell every live process which batches have a request out, so that expiry never deletes a
+ * batch someone is still sending: were that request to fail, there would be no file left to retry from.
+ *
+ * A sender holds a shared lock on the batch for as long as its request is out. Expiry deletes a batch only
+ * while holding an exclusive lock on it, which it cannot have while anyone holds the shared one. The kernel
+ * drops a lock when its process dies, so a batch whose sender is gone is expired like any other.
+ *
+ * A POSIX lock belongs to the process rather than to the descriptor it was taken through, and closing any
+ * descriptor this process has on the file releases it. So the batches locked here are remembered, and expiry
+ * in this process asks that rather than opening a descriptor on one. The lock is taken after the batch has
+ * been read for sending, since the read closes a descriptor of its own; a batch expired by another process in
+ * the moment between is still sent from what was read, but has no file for a retry.
+ */
+final class BatchLocks {
+ /** A batch this process is sending, the descriptor its lock was taken through, and how many send it. */
+ private static final class Held {
+ final RandomAccessFile file;
+ FileLock lock;
+ int senders;
+
+ Held(RandomAccessFile file) {
+ this.file = file;
+ }
+ }
+
+ /**
+ * The batches of one directory this process has a request out for, by name. Its monitor guards the probes
+ * and deletions below as well, so that a store in this process cannot start sending a batch another is
+ * expiring. One per directory rather than one for all, since a lock taken or dropped on one file leaves
+ * every other file's alone, so stores of different environments have nothing to wait on each other for.
+ */
+ private static final class Directory {
+ final Map
+ * The mobile key and the process are both part of where it writes, so an application configured
+ * for several environments, or running the SDK in several processes, gets one store per
+ * combination and they never touch each other's events.
+ */
+ private EventStore makeEventStore(ClientContext clientContext, ClientContextImpl impl) {
+ return EventStore.create(
+ impl.getPlatformState(),
+ clientContext.getMobileKey(),
+ capacity,
+ shareOfCapacityBytes(clientContext.getConfig()),
+ maxAgeMillis,
+ eventPersistence != EventPersistence.DISABLED,
+ clientContext.getBaseLogger());
+ }
+
+ /**
+ * This environment's share of the byte capacity, which is for every environment the application configured
+ * together, where the event capacity is for each one.
+ */
+ private long shareOfCapacityBytes(LDConfig config) {
+ int environments = config == null ? 1 : Math.max(1, config.getMobileKeys().size());
+ // At least one byte, since the store reads zero as the default.
+ return Math.max(1, capacityBytes / environments);
+ }
+
@Override
public LDValue describeConfiguration(ClientContext clientContext) {
return LDValue.buildObject()
.put("allAttributesPrivate", allAttributesPrivate)
.put("diagnosticRecordingIntervalMillis", diagnosticRecordingIntervalMillis)
.put("eventsCapacity", capacity)
- .put("diagnosticRecordingIntervalMillis", diagnosticRecordingIntervalMillis)
.put("eventsFlushIntervalMillis", flushIntervalMillis)
.build();
}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DeliveryRetryState.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DeliveryRetryState.java
new file mode 100644
index 000000000..fdb317064
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DeliveryRetryState.java
@@ -0,0 +1,142 @@
+package com.launchdarkly.sdk.android;
+
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * When event delivery may post again after a failure, following the SDK Retry Behavior specification as
+ * event durability §9 binds it.
+ *
+ * A failure never stops delivery, it only makes the next post wait. An ordinary one, such as a 503 or no
+ * network, waits from a second up to the flush interval, so it is never later than the schedule already
+ * would be. A refusal that is unlikely to pass soon, such as a 401, waits about eight and a half minutes,
+ * doubling up to an hour, rather than deleting the batch: a valid key is sometimes refused for minutes
+ * during an incident, and every event the fleet recorded meanwhile would be lost. A {@code Retry-After}
+ * sets the wait instead. One accepted delivery puts everything back.
+ *
+ * One per event processor, so each environment backs off on its own. Held only in memory, so a new run of
+ * the application posts at once.
+ */
+final class DeliveryRetryState {
+ /** The longest a {@code Retry-After} may make delivery wait, so a bad value cannot look like stopping. */
+ static final long MAX_SERVER_DIRECTED_WAIT_MILLIS = TimeUnit.HOURS.toMillis(1);
+
+ /** The ceiling once a refusal has been seen, until a delivery is accepted. */
+ static final long EXTENDED_CEILING_MILLIS = TimeUnit.HOURS.toMillis(1);
+
+ /**
+ * Where the attempt count goes at least to on a refusal, so its first wait is 2^9 initial delays, about
+ * eight and a half minutes, rather than starting from a second.
+ */
+ static final int ATTEMPTS_AFTER_UNEXPECTED = 10;
+
+ private final long initialDelayMillis;
+ private final long normalCeilingMillis;
+ private final long extendedCeilingMillis;
+ private final Random random;
+
+ // Guarded by this.
+ private int attempts;
+ private long maxDelayMillis;
+ private boolean waiting;
+ private long nextAttemptNanos;
+
+ /**
+ * @param flushIntervalMillis the normal ceiling; never below the initial delay, so a short interval does
+ * not make a failing service be posted to more often than once a second
+ */
+ DeliveryRetryState(long flushIntervalMillis) {
+ this(DirectEventProcessor.RETRY_DELAY_MILLIS, Math.max(flushIntervalMillis,
+ DirectEventProcessor.RETRY_DELAY_MILLIS), EXTENDED_CEILING_MILLIS, new Random());
+ }
+
+ DeliveryRetryState(long initialDelayMillis, long normalCeilingMillis, long extendedCeilingMillis,
+ Random random) {
+ this.initialDelayMillis = initialDelayMillis;
+ this.normalCeilingMillis = normalCeilingMillis;
+ this.extendedCeilingMillis = extendedCeilingMillis;
+ this.random = random;
+ this.maxDelayMillis = normalCeilingMillis;
+ }
+
+ /** The wait after a first failure, which a flush waiting on the delivery that failed sits out. */
+ long initialDelayMillis() {
+ return initialDelayMillis;
+ }
+
+ /** An accepted delivery, which is all it takes for the next failure to start from a second again. */
+ synchronized void recordSuccess() {
+ attempts = 0;
+ maxDelayMillis = normalCeilingMillis;
+ waiting = false;
+ }
+
+ /**
+ * @param unexpected whether the failure is unlikely to pass soon: a 4xx other than 400, 408, 413, 415,
+ * 422 and 429, or
+ * a TLS or certificate failure
+ * @param retryAfterMillis the wait the response asked for, or a negative number where it gave none
+ * @return how long nothing may be posted
+ */
+ synchronized long recordFailure(boolean unexpected, long retryAfterMillis) {
+ if (unexpected) {
+ attempts = Math.max(attempts + 1, ATTEMPTS_AFTER_UNEXPECTED);
+ maxDelayMillis = extendedCeilingMillis;
+ } else {
+ attempts++;
+ }
+ long waitMillis;
+ if (retryAfterMillis >= 0) {
+ // Every instance hears the same value at the same moment, so it is jittered to keep them from
+ // posting together when it runs out.
+ long directed = Math.min(retryAfterMillis, MAX_SERVER_DIRECTED_WAIT_MILLIS);
+ waitMillis = directed - (long) (random.nextDouble() * (directed / 2.0));
+ } else {
+ // Not jittered: each instance's failures follow its own history, which already spreads them.
+ int doublings = Math.min(attempts - 1, 40);
+ waitMillis = Math.min(maxDelayMillis, initialDelayMillis << doublings);
+ }
+ waiting = true;
+ nextAttemptNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(waitMillis);
+ return waitMillis;
+ }
+
+ /**
+ * A failure of a post that went through a wait, as close() and {@code flushAndWait} may: an extra try,
+ * not a step of the backoff. It leaves the attempt count and the ceiling as they were, so a caller
+ * flushing through a refusal several times does not push flush, the periodic delivery and the retry out
+ * to the hour. A {@code Retry-After} can still make the wait longer, never shorter. Once the wait has run
+ * out it is an ordinary failure.
+ *
+ * @param unexpected as {@link #recordFailure}
+ * @param retryAfterMillis as {@link #recordFailure}
+ */
+ synchronized void recordFailureDuringWait(boolean unexpected, long retryAfterMillis) {
+ if (remainingWaitNanos() == 0) {
+ recordFailure(unexpected, retryAfterMillis);
+ return;
+ }
+ if (retryAfterMillis < 0) {
+ return;
+ }
+ long directed = Math.min(retryAfterMillis, MAX_SERVER_DIRECTED_WAIT_MILLIS);
+ long waitMillis = directed - (long) (random.nextDouble() * (directed / 2.0));
+ long asked = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(waitMillis);
+ if (asked - nextAttemptNanos > 0) {
+ nextAttemptNanos = asked;
+ }
+ }
+
+ /** How long until a post is allowed, in nanoseconds; zero once it is. */
+ synchronized long remainingWaitNanos() {
+ if (!waiting) {
+ return 0;
+ }
+ long remaining = nextAttemptNanos - System.nanoTime();
+ if (remaining <= 0) {
+ waiting = false;
+ return 0;
+ }
+ return remaining;
+ }
+}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiagnosticReporter.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiagnosticReporter.java
new file mode 100644
index 000000000..bed4612f0
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiagnosticReporter.java
@@ -0,0 +1,229 @@
+package com.launchdarkly.sdk.android;
+
+import com.launchdarkly.logging.LDLogger;
+import com.launchdarkly.sdk.internal.events.DiagnosticEvent;
+import com.launchdarkly.sdk.internal.events.DiagnosticStore;
+import com.launchdarkly.sdk.internal.events.EventSender;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Sends the SDK's diagnostic events: the init event once per process, and the statistics once per recording
+ * interval, only while the SDK is online and in the foreground.
+ *
+ * Part of {@link DirectEventProcessor}, which tells it through {@link #setMode} when that changes and when the
+ * launch gate the init event waits for opens. Diagnostics are best-effort telemetry about the SDK, so nothing
+ * here is stored or retried. Posts go out one at a time on a thread of their own, never the one delivering
+ * analytics events.
+ */
+final class DiagnosticReporter {
+ /** What the reporter needs from the event processor whose events it describes. */
+ interface Processor {
+ /** @return the events dropped since this was last called; see {@link DirectEventProcessor#getAndClearDroppedCount()} */
+ long getAndClearDroppedCount();
+
+ /** Hears the service's clock, which a diagnostic response carries as an analytics one does. */
+ void recordPastTime(long timeFromServer);
+ }
+
+ private final DiagnosticStore store;
+ private final EventSender sender;
+ private final URI eventsUri;
+ private final long recordingIntervalMillis;
+ /** Where the statistics task runs: the event processor's scheduler, which only hands each post on. */
+ private final ScheduledExecutorService scheduler;
+ private final ExecutorService postExecutor;
+ private final Processor processor;
+ private final LDLogger logger;
+
+ /** Only touched in {@link #setMode}, which the event processor calls under its state lock. */
+ private ScheduledFuture statsTask;
+
+ /** Whether diagnostics may go out now. Written in {@link #setMode}, read on the posting thread. */
+ private volatile boolean enabled;
+ private volatile boolean initSent = false;
+
+ /**
+ * True while a diagnostic event is on its way to the service, so that a later one is dropped
+ * rather than queued behind it.
+ */
+ private final AtomicBoolean postInFlight = new AtomicBoolean(false);
+
+ /**
+ * @param store the diagnostics to report, or null where the application opted out of them
+ */
+ DiagnosticReporter(
+ DiagnosticStore store,
+ EventSender sender,
+ URI eventsUri,
+ long recordingIntervalMillis,
+ ScheduledExecutorService scheduler,
+ ExecutorService postExecutor,
+ Processor processor,
+ LDLogger logger
+ ) {
+ this.store = store;
+ this.sender = sender;
+ this.eventsUri = eventsUri;
+ this.recordingIntervalMillis = recordingIntervalMillis;
+ this.scheduler = scheduler;
+ this.postExecutor = postExecutor;
+ this.processor = processor;
+ this.logger = logger;
+ }
+
+ /**
+ * Starts or stops reporting, and sends the init event once it may go. Called under the event processor's
+ * state lock, so calls arrive in the order the changes happened.
+ *
+ * @param online whether the event processor is open, online and in the foreground
+ * @param launchGateOpen whether the launch gate has opened, which the init event waits for
+ */
+ void setMode(boolean online, boolean launchGateOpen) {
+ enabled = online && store != null;
+ statsTask = EventUtil.enableOrDisableTask(scheduler, enabled, statsTask, recordingIntervalMillis,
+ guarded(this::sendStats));
+ if (enabled && launchGateOpen && !initSent && claimPost()) {
+ DiagnosticStore store = this.store;
+ post(() -> {
+ // Re-checked on the posting thread: going online and coming to the foreground are
+ // two separate calls, and both want to send the init event we never got to send.
+ // Suspension is re-checked for the same reason it is for the periodic event, and
+ // costs nothing here: getInitEvent consumes nothing, so a skipped init is simply
+ // built again the next time diagnostics are enabled.
+ if (!initSent && enabled) {
+ send(store.getInitEvent(), true);
+ }
+ });
+ }
+ }
+
+ /** Counts a batch of analytics events sent, for the next statistics event. */
+ void recordEventsInBatch(int eventCount) {
+ if (store != null) {
+ store.recordEventsInBatch(eventCount);
+ }
+ }
+
+ /**
+ * Releases the sender on the posting thread, after any post still on it, and stops that thread taking
+ * more. The statistics task is canceled by the {@link #setMode} that closing the event processor makes.
+ */
+ void close() {
+ try {
+ postExecutor.submit(guarded(this::closeSender));
+ } catch (RuntimeException e) { // the executor was shut down under us
+ closeSender(); // so nothing can still be using it
+ }
+ postExecutor.shutdown();
+ }
+
+ private void closeSender() {
+ try {
+ sender.close();
+ } catch (IOException e) {
+ logUnexpectedError(e);
+ }
+ }
+
+ private void sendStats() {
+ // Diagnostics are not sent while offline or in the background. Canceling the task when either
+ // becomes true is not enough on its own: canceling does not stop a run already underway, and
+ // the init event is submitted before it reaches the executor. Either can get here after the
+ // state changed, so both test enabled again.
+ if (!enabled || !claimPost()) {
+ return;
+ }
+ DiagnosticStore store = this.store;
+ post(() -> {
+ // The check that decides the outcome, and it has to come before the event is built rather
+ // than after. createEventAndReset hands back the period and clears it: the stream init
+ // records, the events-in-batch count and the period start all move. getAndClearDroppedCount
+ // does the same for the dropped count. This runs on the posting thread, which may have
+ // been busy with an earlier post for as long as that post took, so the state can easily
+ // have changed since the check above. Bailing out after the event was built would discard
+ // a period outright and leave the next event describing a window that begins after the
+ // reset. Bailing out here leaves everything where it is, for the next period to carry.
+ if (!enabled) {
+ return;
+ }
+ send(store.createEventAndReset(processor.getAndClearDroppedCount(), 0), false);
+ });
+ }
+
+ /**
+ * Posts an event that has already been built. Requires the caller to have checked {@link #enabled},
+ * because for the periodic event building it is what consumes the period.
+ */
+ private void send(DiagnosticEvent event, boolean isInit) {
+ try {
+ byte[] data = event.getJsonValue().toJsonString().getBytes(StandardCharsets.UTF_8);
+ EventSender.Result result = sender.sendDiagnosticEvent(data, eventsUri);
+ // Only read for the service's clock. One the service refused stops nothing: analytics delivery
+ // backs off on its own responses, and a diagnostic is not retried anyway.
+ if (result != null && result.getTimeFromServer() != null) {
+ processor.recordPastTime(result.getTimeFromServer().getTime());
+ }
+ if (isInit) {
+ // Attempted, not delivered. A failed post gives back an unsuccessful Result rather
+ // than throwing, so this marks the init event done either way and the process never
+ // retries it. That matches DefaultEventProcessor, which is the behavior to keep:
+ // diagnostics are best-effort telemetry about the SDK, and a retry that outlived
+ // its own init would describe a configuration the application has moved on from.
+ initSent = true;
+ }
+ } catch (Exception e) {
+ logUnexpectedError(e);
+ }
+ }
+
+ /**
+ * Takes the posting thread, or reports that the last event is still on it.
+ *
+ * A new event is dropped rather than queued behind the old one. Diagnostics are best-effort
+ * telemetry, and a post can take tens of seconds against a network that never answers. Queuing
+ * would let an outage accumulate events describing an SDK state the application has since moved
+ * past. The same reasoning makes a failed init event final rather than retried.
+ */
+ private boolean claimPost() {
+ if (postInFlight.compareAndSet(false, true)) {
+ return true;
+ }
+ logger.debug("Skipped a diagnostic event because the previous one is still being posted");
+ return false;
+ }
+
+ /**
+ * Hands a claimed post to the posting thread, releasing the claim once it ends.
+ *
+ * @param post what to run there, which must hold a claim from {@link #claimPost()}
+ */
+ private void post(Runnable post) {
+ Runnable releasing = () -> {
+ try {
+ post.run();
+ } finally {
+ postInFlight.set(false);
+ }
+ };
+ try {
+ postExecutor.submit(guarded(releasing));
+ } catch (RuntimeException e) { // the executor was shut down under us
+ postInFlight.set(false);
+ }
+ }
+
+ private Runnable guarded(Runnable task) {
+ return EventUtil.guarded(logger, task);
+ }
+
+ private void logUnexpectedError(Throwable e) {
+ EventUtil.logUnexpectedError(logger, e);
+ }
+}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java
index 89d0da48b..7126c0f94 100644
--- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java
@@ -6,19 +6,17 @@
import com.launchdarkly.sdk.LDContext;
import com.launchdarkly.sdk.LDValue;
import com.launchdarkly.sdk.android.subsystems.EventProcessor;
-import com.launchdarkly.sdk.internal.events.DiagnosticEvent;
import com.launchdarkly.sdk.internal.events.DiagnosticStore;
import com.launchdarkly.sdk.internal.events.Event;
import com.launchdarkly.sdk.internal.events.EventSender;
-import com.launchdarkly.sdk.internal.events.EventSummarizer;
-import com.launchdarkly.sdk.internal.events.Sampler;
import java.io.IOException;
import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -27,177 +25,315 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
/**
* The Android SDK's analytics event processor.
*
- * Recording an event summarizes it immediately and, only if it has to be delivered in full,
- * buffers it. There is no queue between the calling thread and the summarizer, so a burst of flag
- * evaluations with trackEvent = false cannot displace anything and costs a counter increment,
- * The configured capacity limits only the events that have trackEvents = true.
+ * What the application records is accepted, held and committed to the store by {@link EventCommitter}, which
+ * says which calls are commit points and which thread each commit runs on. This class delivers what is
+ * committed, and runs the processor's life: flushes, going offline and into the background, and close.
+ *
+ * Threads. Every delivery runs on {@link #scheduler}, a single thread, queued there by {@link EventDeliveryScheduler}:
+ * a requested one, the periodic one, the retry after a failure, and the final one {@code close()} queues. So no two deliveries of this processor overlap, and the final one
+ * waits behind a post or retry already running there, within the budget {@code close()} gives it.
+ * Diagnostic events go out through {@link DiagnosticReporter}, on a thread of their own.
+ *
+ * Overlap is across processes. Two processes of one application read the same directory, so both can send
+ * one stored batch. They send it under the same payload ID, and LaunchDarkly discards the repeat rather than
+ * counting the events twice.
+ *
+ * Flag values come first. Where events persist, what a previous run left waits at launch until a fixed time
+ * after the SDK first goes online, and the deliveries this processor starts on its own send one batch at a time,
+ * a gap apart, leaving room for flag requests. Both go by time alone: nothing here hears what the flag data
+ * source is doing. A delivery the application asks for is never held.
*/
final class DirectEventProcessor implements EventProcessor {
+ /** How long the deliveries this processor starts on its own wait, so that flag values come first. */
+ static final class Pacing {
+ static final Pacing SPECIFIED = new Pacing(2_000, 1_000);
+ /** No waits at all, for tests about something else. */
+ static final Pacing NONE = new Pacing(0, 0);
+
+ /**
+ * How long after the SDK first goes online that a previous run's events, and the diagnostic init event,
+ * wait. It gives the application's startup traffic, the first flag request among it, the network first,
+ * and is short enough for an application that dies a few seconds after every launch to still send them.
+ */
+ final long launchDelayMillis;
+ /** The least time between one paced batch's answer and the next batch's request. */
+ final long gapMillis;
+
+ Pacing(long launchDelayMillis, long gapMillis) {
+ this.launchDelayMillis = launchDelayMillis;
+ this.gapMillis = gapMillis;
+ }
+ }
+
/**
* How long {@link #close()} spends waiting for the final delivery before it gives up and returns.
*
* {@code close()} runs on the caller's thread, which for an application shutting down is usually the
* main one, so this has to stay well inside the five seconds Android allows before an unanswered
- * input event becomes an ANR. Two seconds is where the return on waiting longer falls off: the
- * events client keeps pooled connections for only five seconds against a thirty second flush
- * interval, so this post nearly always pays a full DNS, TCP and TLS handshake -- about three round
- * trips, which two seconds covers up to roughly a 600ms RTT. A network slower than that is one the
- * post is likely to fail on anyway.
+ * input event becomes an ANR. Two seconds is where the return on waiting longer falls off. The
+ * events client keeps pooled connections for only five seconds against a thirty-second flush
+ * interval, so this post nearly always pays a full DNS, TCP and TLS handshake. That is about three
+ * round trips, which two seconds covers up to a round-trip time of roughly 600 ms. A network slower
+ * than that is one the post is likely to fail on anyway.
*
* Overshooting the budget is cheaper than it looks, because the delivery is not canceled when the
* budget expires; see {@link #close()}.
*/
static final long DEFAULT_CLOSE_BUDGET_MILLIS = 2_000;
- private final OutboundEventBuffer buffer;
- private final EventSender eventSender;
+ /**
+ * The first wait of the backoff after a failure, as the other SDKs wait for their one retry. A flush
+ * waiting on the delivery that failed sits out this wait and hears the retry's outcome; any longer
+ * wait it does not, and is told the delivery failed.
+ */
+ static final long RETRY_DELAY_MILLIS = 1_000;
+
+ // Set in the constructor and never changed.
+
+ private final EventStore store;
+ private final EventCommitter committer;
+ private final AnalyticsEventSender analyticsEventSender;
+ /** When the next post may go, after a failure. Consulted before every post, and only there. */
+ private final DeliveryRetryState retryState;
private final URI eventsUri;
- private final DiagnosticStore diagnosticStore;
+ private final DiagnosticReporter diagnostics;
private final long flushIntervalMillis;
- private final long diagnosticRecordingIntervalMillis;
private final long closeBudgetMillis;
private final ScheduledExecutorService scheduler;
- private final ExecutorService diagnosticExecutor;
private final LDLogger logger;
- private final AtomicBoolean inBackground;
- private final AtomicBoolean offline;
- private final AtomicBoolean closed = new AtomicBoolean(false);
- // Set when the service tells us to stop, e.g. because the mobile key is invalid.
- private volatile boolean disabled = false;
- private volatile boolean diagnosticInitSent = false;
- private final AtomicLong lastKnownPastTime = new AtomicLong(0);
+ // Guarded by stateLock.
+ /**
+ * Guards {@link #flushTask} and every {@link DiagnosticReporter#setMode}, and every change to
+ * {@link #inBackground} and {@link #offline}, so the tasks are rescheduled in the order the changes
+ * happen. Those two are atomic so they can be read without it.
+ */
private final Object stateLock = new Object();
private ScheduledFuture flushTask;
- private ScheduledFuture diagnosticTask;
+ private final AtomicBoolean inBackground;
+ private final AtomicBoolean offline;
- /**
- * Full events recorded but not yet encoded, guarded by {@link #recordLock}.
- */
- private final List
- * One evaluation can produce a counter, a full event and a debug event, and the three have to
- * land together. Taken separately, a flush landing between them splits one evaluation across two
- * payloads, and a {@link #close()} landing between them delivers the counter and then refuses the
- * full event -- leaving a summary that says an evaluation happened and no event to go with it.
- *
- * Held for the appends and the handover of the run, never across the encode. Recording runs on
- * whichever thread evaluated a flag, which on Android is usually the main one. The worst it may
- * wait for is another thread's memory operation; if the encoder ran under this lock, an
- * evaluation would instead wait on the dominant cost of the whole path.
+ * Those events are already as old as whatever happened to that run, so they are sent once the launch
+ * gate opens rather than at the next interval. The application in front of the user has moved on; an
+ * error report about a crash is not worth much thirty seconds late, and the process might not live that
+ * long either.
*/
- private final Object recordLock = new Object();
+ private final AtomicBoolean hasEventsFromPreviousRun = new AtomicBoolean(false);
+
+ private final Pacing pacing;
/**
- * How many events the SDK will hold between flushes.
+ * Whether events give way to flag values: only where the application asked for persistence, like the
+ * rest of what persistence brings. Without it there is no previous run to hold back, and delivery is
+ * what it was before persistence: no launch gate and no pacing.
*/
- private final int capacity;
+ private final boolean givesWayToFlags;
- private final AtomicBoolean capacityExceeded = new AtomicBoolean(false);
+ /**
+ * Whether the launch gate has opened. It opens once the launch delay has passed since the SDK first went
+ * online. Until then neither a previous run's events nor the diagnostic init event are sent. Open from the
+ * start where events do not give way. Only set on the scheduler thread.
+ */
+ private volatile boolean launchGateOpen;
- /** Whether the summarizer being full has already been reported, so it is logged once per run. */
- private final AtomicBoolean summaryContextsExceeded = new AtomicBoolean(false);
+ /** Set once, under {@link #stateLock}, when the SDK first goes online; the gate counts from then. */
+ private volatile boolean wentOnline;
+ private volatile long wentOnlineNanos;
+ /** Events dropped for capacity, and evaluations the summarizer turned away, which diagnostics carry. */
private final AtomicLong droppedEvents = new AtomicLong(0);
/**
- * True while a diagnostic event is on its way to the service, so that a later one is dropped
- * rather than queued behind it.
+ * How many events the SDK accepted and then dropped for good.
+ *
+ * A batch the service refuses is kept and retried, so it is not a loss, unless the refusal is a
+ * second refusal of the payload in a row. Otherwise what is lost is what the store never received: an event or a
+ * summary that could not be serialized when it was committed, or that serialized too large to
+ * store. A delivery that
+ * finds nothing to send cannot tell those apart from events that arrived, so without this it
+ * would report success for them.
+ *
+ * A count rather than a flag because a delivery can answer long after it ran: one that met a
+ * retryable failure answers only when its retry does, and another delivery can come and go in
+ * between. Each delivery takes the count as it closes its batch and answers only for the losses
+ * up to that mark, so it is never told about a loss that happened after it had taken everything
+ * it could speak for. Only the order of two values ever matters, never the difference between
+ * them, so a failure that lost several events need only move it once.
+ *
+ * Atomic because commits run on the commit executor and on callers' threads, and a caller
+ * waiting behind a retry is answered from a pooled thread.
*/
- private final AtomicBoolean diagnosticPostInFlight = new AtomicBoolean(false);
+ private final AtomicLong lostEvents = new AtomicLong(0);
/**
- * The two threads that post through {@link #eventSender}, counted down at shutdown so that
- * whichever finishes last is the one that closes it.
+ * The loss count through which losses belong to a delivery some {@link #flushAsync()} caller waits on,
+ * so that each loss is told to one caller: the first whose delivery passed it.
+ *
+ * Claimed as each pass ends, in the order deliveries run, rather than as each answers. A delivery
+ * waiting on a retry answers after the one behind it may have, and claiming on answering would hand
+ * that later delivery the losses before the earlier one's mark, leaving the flush whose delivery lost
+ * the events answering true once its retry works. A delivery nobody waits on claims nothing, so what
+ * it passed is told to the next caller instead of to no one.
+ *
+ * Only read and written on the scheduler thread, which runs every delivery.
*/
- private final AtomicInteger sendersStillDraining = new AtomicInteger(2);
+ private long claimedLostEvents;
/**
- * Guards the handover from running to shut down: held for the length of a submit, and by
- * {@link #close()} while it queues the release of the sender and stops the executors accepting
- * work. Nothing blocking happens under it.
+ * How many refusals of the payload in a row drop a batch: a 400, 413, 415 or 422, in any mix
+ * ({@link LDUtil#isEventPayloadRefused}). Each says the service will not take this body, which posting
+ * it again will not change, and a kept batch is the oldest and so the next posted: kept for good, it
+ * would hold every batch behind it. Two rather than one, so one from a proxy or an incident costs nothing.
*/
- private final Object submitLock = new Object();
+ static final int BAD_REQUESTS_BEFORE_DROPPING = 2;
- /** Set under {@link #submitLock} once close() has queued the release of the sender. */
- private boolean shuttingDown = false;
+ // The batch that last had its payload refused and how many times in a row: delivery goes oldest first
+ // and stops at a kept failure, so the next post is that batch's unless capacity dropped it. Any response
+ // other than a refusal ends the run; a post that got no response leaves it. Held only in memory, so a new
+ // run of the application counts again.
+ private final Object badRequestLock = new Object();
+ private String badRequestPayloadId;
+ private int badRequestsInARow;
/**
- * Guards {@link #pendingFlush} and {@link #pendingFlushAnswersACaller}. Taken on a caller's
- * thread and on the delivery thread, never while holding {@link #recordLock}, and nothing
- * blocking happens under it.
+ * How many counted failed reads in a row drop a batch. A read that fails is mostly one that will go on
+ * failing, a permission taken away or a directory named like a batch, and expiry cannot delete a file it
+ * cannot open, so the batch would be read, and fail, at every delivery for good. More than one, and
+ * counted at most once per {@link #readFailureSpacingNanos}, so a passing failure such as running out of
+ * file descriptors costs nothing. A read short of memory never counts.
*/
- private final Object flushLock = new Object();
+ static final int READ_FAILURES_BEFORE_DROPPING = 3;
/**
- * The delivery that is queued but has not started, which a flush request arriving now can wait
- * on instead of queueing another. Null while nothing is queued, and cleared again as the queued
- * delivery begins, which is the point past which it can no longer speak for what is recorded.
+ * The least time between two failed reads of a batch that both count toward dropping it: the flush
+ * interval, so a drop takes failures across at least two of them. Every pass that reaches the batch reads
+ * it, and passes can come milliseconds apart, from flushes called in a row or {@code flushAndWait} posting
+ * through the backoff; counted by pass, a failure lasting a moment would drop an intact batch. A failed read
+ * closer than this to the last counted one passes the batch by without counting. Not final only for tests.
*/
- private LDAwaitFuture
- * Without it, a delivery that finds the buffer empty reports success even when the events it is
- * being asked about were taken a moment earlier by another delivery that then lost them: the
- * periodic flush, or a flush whose caller did not wait for the outcome. Only touched on the
- * scheduler thread, which every delivery runs on.
- */
- private boolean eventsLostSinceLastAnswer;
+ private static final class ReadFailures {
+ int counted;
+ long lastCountedNanos;
+ }
DirectEventProcessor(
- OutboundEventBuffer buffer,
- EventSender eventSender,
+ OutboundEventBuffer eventBuffer,
+ EventStore store,
+ EventSender diagnosticEventSender,
+ AnalyticsEventSender analyticsEventSender,
+ DeliveryRetryState retryState,
URI eventsUri,
DiagnosticStore diagnosticStore,
int capacity,
+ boolean commitOnCallerThread,
long flushIntervalMillis,
long diagnosticRecordingIntervalMillis,
long closeBudgetMillis,
+ Pacing pacing,
boolean initiallyInBackground,
boolean initiallyOffline,
ScheduledExecutorService scheduler,
+ ExecutorService commitExecutor,
ExecutorService diagnosticExecutor,
LDLogger logger
) {
- this.buffer = buffer;
- this.eventSender = eventSender;
+ this.store = store;
+ this.analyticsEventSender = analyticsEventSender;
+ this.retryState = retryState;
this.eventsUri = eventsUri;
- this.diagnosticStore = diagnosticStore;
- this.capacity = capacity;
this.flushIntervalMillis = flushIntervalMillis;
- this.diagnosticRecordingIntervalMillis = diagnosticRecordingIntervalMillis;
+ this.readFailureSpacingNanos = TimeUnit.MILLISECONDS.toNanos(flushIntervalMillis);
this.closeBudgetMillis = closeBudgetMillis;
+ this.pacing = pacing;
+ this.givesWayToFlags = store.isPersisting();
+ this.launchGateOpen = !givesWayToFlags;
this.scheduler = scheduler;
- this.diagnosticExecutor = diagnosticExecutor;
this.logger = logger;
+ this.committer = new EventCommitter(eventBuffer, store, capacity, commitOnCallerThread, commitExecutor,
+ closed, lostEvents, droppedEvents, logger);
this.inBackground = new AtomicBoolean(initiallyInBackground);
this.offline = new AtomicBoolean(initiallyOffline);
+ this.deliveryScheduler = new EventDeliveryScheduler(scheduler, givesWayToFlags, pacing.gapMillis,
+ () -> closed.get() || offline.get(), logger);
+ this.diagnostics = new DiagnosticReporter(diagnosticStore, diagnosticEventSender, eventsUri,
+ diagnosticRecordingIntervalMillis, scheduler, diagnosticExecutor,
+ new DiagnosticReporter.Processor() {
+ @Override
+ public long getAndClearDroppedCount() {
+ return DirectEventProcessor.this.getAndClearDroppedCount();
+ }
+
+ @Override
+ public void recordPastTime(long timeFromServer) {
+ committer.recordPastTime(timeFromServer);
+ }
+ },
+ logger);
+
+ // Off the calling thread: this is on the path of LDClient.init, and it reads the filesystem. Only where
+ // events persist, since memory holds nothing from a previous run.
+ if (store.isPersisting()) {
+ deliveryScheduler.submit(this::recoverEventsFromPreviousRun);
+ }
synchronized (stateLock) {
updateScheduledTasks(initiallyInBackground, initiallyOffline);
}
}
+ private void recoverEventsFromPreviousRun() {
+ boolean found;
+ // Below the threads loading flags, which matter more at launch than what a previous run left.
+ Thread thread = Thread.currentThread();
+ int priority = thread.getPriority();
+ thread.setPriority(Thread.MIN_PRIORITY);
+ try {
+ // Lost as a damaged batch is, and told the same way: to the next flush a caller waits on.
+ lostEvents.addAndGet(store.preparePayloadsFromInterruptedLogs());
+ reportDiscarded(store.discardExpired());
+ found = !store.pendingBatches().isEmpty();
+ } finally {
+ thread.setPriority(priority);
+ }
+ if (!found) {
+ return;
+ }
+ // Raised before the gate and offline are read, and claimed by whichever side sees both: opening
+ // the gate, and going online after it, write their state and then claim the flag, so at least one
+ // of the two finds the other's write, and the claim makes it exactly one. Reading them first left
+ // a window where the other side found no flag yet and this then found the SDK still offline, and
+ // neither sent them.
+ hasEventsFromPreviousRun.set(true);
+ if (launchGateOpen && !offline.get() && !isStopped()
+ && hasEventsFromPreviousRun.compareAndSet(true, false)) {
+ deliverPayload();
+ }
+ }
+
@Override
public void recordEvaluationEvent(
LDContext context,
@@ -210,42 +346,8 @@ public void recordEvaluationEvent(
boolean requireFullEvent,
Long debugEventsUntilDate
) {
- try {
- if (isStopped() || context == null) {
- return;
- }
- Event.FeatureRequest event = new Event.FeatureRequest(System.currentTimeMillis(), flagKey,
- context, flagVersion, variation, value, defaultValue, reason, null,
- requireFullEvent, debugEventsUntilDate, false);
- // Built before the lock is taken, so that the critical section is only the writes.
- Event debugEvent = shouldDebugEvent(debugEventsUntilDate) ? event.toDebugEvent() : null;
- boolean contextsExceeded;
- boolean warnContextsExceeded;
- synchronized (recordLock) {
- if (closed.get()) {
- return;
- }
- contextsExceeded = !buffer.summarize(event);
- // Claimed under the lock that the delivery resets it under, so the warning belongs to
- // the run whose summarizer turned this evaluation away rather than to the next one.
- warnContextsExceeded = contextsExceeded
- && summaryContextsExceeded.compareAndSet(false, true);
- if (requireFullEvent) {
- addPending(event);
- }
- if (debugEvent != null) {
- addPending(debugEvent);
- }
- }
- if (contextsExceeded) {
- reportContextsExceeded(warnContextsExceeded);
- }
- } catch (RuntimeException e) {
- // This runs on the application's thread, usually inside a flag evaluation, and an
- // analytics failure must not become the application's failure. Errors such as
- // OutOfMemoryError are left to propagate, so the application's crash reporting sees them.
- logUnexpectedError(e);
- }
+ committer.recordEvaluation(context, flagKey, flagVersion, variation, value, reason, defaultValue,
+ requireFullEvent, debugEventsUntilDate);
}
@Override
@@ -253,7 +355,7 @@ public void recordIdentifyEvent(LDContext context) {
if (isStopped() || context == null) {
return;
}
- record(new Event.Identify(System.currentTimeMillis(), context));
+ committer.recordAtCommitPoint(new Event.Identify(System.currentTimeMillis(), context));
}
@Override
@@ -261,75 +363,52 @@ public void recordCustomEvent(LDContext context, String eventKey, LDValue data,
if (isStopped() || context == null) {
return;
}
- record(new Event.Custom(System.currentTimeMillis(), eventKey, context, data, metricValue));
+ committer.recordAtCommitPoint(new Event.Custom(System.currentTimeMillis(), eventKey, context, data,
+ metricValue));
+ }
+
+ /** As {@link EventCommitter#record}. */
+ long record(Event event) {
+ return committer.record(event);
}
/**
- * Holds an event for the next flush to encode, counting it as dropped if the SDK is already full.
- *
- * Capacity is consulted before anything is encoded, so an event that will not be kept is never
- * encoded.
- * That ordering is what bounds an application re-evaluating a tracked flag in a render loop: once
- * the limit is reached the cost of an evaluation falls back to its summary counter, however fast
- * the loop runs.
+ * Records a batch lost for good. The delivery that lost it fails, but a delivery nobody waits on has no
+ * caller to tell: the periodic one, a {@code flush()}, or the one going online starts. So the loss is
+ * also counted for the next {@link #flushAsync()} to hear about. A caller waiting on this delivery can
+ * then hear about it from the next one too.
*/
- void record(Event event) {
- try {
- synchronized (recordLock) {
- // The close check that decides the outcome, as against the fast path the public record
- // methods take before building the event. deliverPayload lifts the run out under this
- // same lock, so testing the flag here orders a record against close()'s final delivery:
- // either the event is in the list before that delivery takes it, or it is refused.
- // Tested outside the lock the two interleave, and an event can be left in a list that
- // nothing will drain again.
- if (closed.get()) {
- return;
- }
- addPending(event);
- }
- } catch (RuntimeException e) {
- // As in recordEvaluationEvent: on the caller's thread, so a failure is logged, not thrown.
- logUnexpectedError(e);
- }
+ private void reportBatchLost(EventStore.Batch batch) {
+ lostEvents.addAndGet(batch.eventCount);
}
/**
- * Holds one event for the next flush, unless it was sampled out or there is no room. Requires
- * {@link #recordLock}.
+ * Records stored events discarded for being older than the maximum age.
+ *
+ * Counted as dropped rather than lost: a flush is about the caller's events, and the ones discarded
+ * are long past anything a caller is waiting on.
*/
- private void addPending(Event event) {
- // Ahead of the capacity check, because an event the SDK was never going to send is not a
- // loss and must not be counted as one. Sampling and capacity are different reasons not to
- // keep an event, and only the second is one the SDK owes anyone a count of.
- if (!Sampler.shouldSample(event.getSamplingRatio())) {
+ private void reportDiscarded(int events) {
+ if (events <= 0) {
return;
}
- if (pending.size() >= capacity) {
- if (capacityExceeded.compareAndSet(false, true)) {
- logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events.");
- }
- droppedEvents.incrementAndGet();
- return;
- }
- capacityExceeded.set(false);
- pending.add(event);
+ logger.debug("Discarded {} stored event(s) older than the maximum age", events);
+ droppedEvents.addAndGet(events);
}
/**
- * Counts an evaluation the summarizer turned away, which it does when counting it would have meant
- * holding a context beyond the configured capacity.
- *
- * A refused evaluation is a loss in the same sense a refused event is -- nothing later reconstructs
- * a counter -- so it is reported the same way, through the dropped count diagnostics carry.
+ * Claims the losses up to a mark for the delivery that just ended its pass, which a caller waits on.
+ * Runs on the scheduler thread.
+ *
+ * @param mark the loss count as of the close of that delivery's batch
+ * @return true if a loss at or before the mark belonged to no earlier delivery a caller waited on
*/
- private void reportContextsExceeded(boolean warn) {
- // Warned once per delivery run rather than once per process: the summarizer's contexts are
- // cleared with each run, so a later overflow is a new one worth hearing about.
- if (warn) {
- logger.warn("Exceeded the number of contexts that can be summarized at once." +
- " Increase capacity to avoid dropping evaluations.");
+ private boolean claimEventsLost(long mark) {
+ if (mark <= claimedLostEvents) {
+ return false;
}
- droppedEvents.incrementAndGet();
+ claimedLostEvents = mark;
+ return true;
}
/**
@@ -358,6 +437,7 @@ public void setOffline(boolean offline) {
}
updateScheduledTasks(inBackground.get(), offline);
}
+ deliveryScheduler.wakePacedDelivery();
}
@Override
@@ -365,7 +445,8 @@ public void flush() {
if (isStopped()) {
return;
}
- queueDelivery(false);
+ committer.commitBeforeDelivery();
+ queueDelivery(false, false);
}
@Override
@@ -373,7 +454,8 @@ public void blockingFlush() {
if (isStopped()) {
return;
}
- Future
- * A delivery that has not started yet will send everything recorded before it starts. That
- * includes the caller's events, so waiting on it is as good as queuing a new one. Without this,
- * flushes arriving faster than a post completes each queue their own, and the one that matters --
- * the {@code flushAndWait} at shutdown -- waits behind all of them.
+ * {@link #flushAsync()} that bypasses the backoff: it posts whatever wait the backoff is in, as close()
+ * does. For a caller about to lose the ability to send the events, such as an uncaught exception
+ * handler: {@code LDClient.flushAndWait}, and {@code LDCrashHandler} through it. After a refusal that wait
+ * is minutes to an hour, and where nothing persists the events would not outlive it. A failure it meets
+ * there leaves the wait as it was. The caller's timeout bounds how long it waits for the answer.
+ */
+ Future
- * Deliveries run one at a time, so any delivery that was in flight when a caller asked has
- * finished before this one starts, and has already recorded whether it lost what it took. A
- * caller who will hear the answer is told no if anything was lost since the last caller was
- * told, as well as if this delivery fails: an empty buffer is not evidence that the events
- * which used to be in it arrived.
- */
- private void runDelivery(LDAwaitFuture
+ * Paced where events give way to flags, since nobody asked for it now: see {@link EventDeliveryRun#awaitTurn()}.
+ */
+ private void deliverPayload() {
+ commitAndDeliverReportingOutcome(deliveryScheduler.ownRun());
}
/**
- * Serializes and sends everything buffered, for the periodic flush, which has nobody waiting to
- * find out how it went. It is a fixed-delay series, so a run is only ever scheduled once the one
- * before it has finished and these cannot pile up the way requested flushes could.
+ * The periodic flush, on the scheduler thread. Until the launch gate opens it only commits, where events persist:
+ * a delivery sends the oldest batches first, which are what a previous run left, so one now would send them ahead
+ * of the gate. What this run recorded goes with the first delivery after the gate opens.
*/
- private void deliverPayload() {
- deliverPayloadReportingOutcome();
+ private void flushIntervalEnded() {
+ if (!launchGateOpen) {
+ if (store.isPersisting()) {
+ committer.commitDurably();
+ }
+ return;
+ }
+ deliverPayload();
}
/**
- * Delivers as {@link #deliverPayload()} does, and says whether it worked, for the callers of a
- * requested flush, who are waiting to find out.
+ * Writes everything accepted so far, then delivers.
*
- * Runs on the scheduler thread, which is single-threaded, so only one payload is ever in flight
- * and the run is taken exactly once per delivery. The run and the counters are taken together
- * under {@link #recordLock}, so an evaluation is never split across two payloads, and encoded
- * outside it, so recording does not wait on the encoder.
- *
- * @return true if the events reached the service, or if there were none to send; false if they
- * could not be sent, the service did not accept them, or some could not be serialized
+ * This commit looks redundant with the one in {@link EventCommitter#commitAndCloseBatch()}, and online it is: the
+ * second finds nothing pending and returns. It is kept for the case where it is the only one. A
+ * delivery refused for being offline returns before {@code commitAndCloseBatch}. Without
+ * this, a flush while offline would leave everything recorded in memory, and a process that then
+ * died would lose it -- the case persistence exists for. The periodic flush needs it most: evaluations
+ * short of a full pending run, with no track or identify among them, are otherwise committed by
+ * nothing for as long as an outage lasts.
+ *
+ * Only where events persist. Otherwise the commit makes nothing durable: online, the one in
+ * {@code commitAndCloseBatch} is the drain, and offline it would encode events that cannot be sent
+ * into bytes that wait in memory exactly as the events would have.
*/
- private boolean deliverPayloadReportingOutcome() {
- if (disabled || offline.get()) {
- // Nothing is taken, so nothing is lost: the events stay buffered for a later delivery.
- return false;
- }
- List
+ * The retry is handed back for the caller to wait on only where it comes after the first wait. Any
+ * longer one, a refusal's or one grown through an outage, is the backoff saying not to post for a
+ * while, and a flush that sat it out would hold its caller for minutes to say what it can say now.
+ *
+ * Runs on the scheduler thread, as every delivery does, so no two overlap; see the class
+ * documentation.
+ */
+ private DeliveryOutcome deliverPayloadAndRetryLater(EventDeliveryRun run) {
+ if (offline.get()) {
+ return DeliveryOutcome.CLOSED_NOTHING;
}
- if (diagnosticStore != null) {
- diagnosticStore.recordEventsInBatch(payload.getEventCount());
+ DeliveryPass pass = deliverWaitingThenNew(run);
+ if (pass.outcome.kept()) {
+ // A batch kept for a reason other than the network, such as a read that failed, has no wait of its
+ // own, and is retried after the first one.
+ long waitNanos = retryState.remainingWaitNanos();
+ long delayNanos = waitNanos > 0 ? waitNanos
+ : TimeUnit.MILLISECONDS.toNanos(retryState.initialDelayMillis());
+ LDAwaitFuture
+ * Closing first would add a batch for every pass a failure lasts: every flush and every periodic
+ * delivery of an outage would leave one more file, or one more batch held in memory, each with the
+ * summary its commit wrote. Held back, what is committed meanwhile goes on into the open log, which is
+ * closed once it is full or once the batches ahead of it are gone, so an outage costs a batch per
+ * {@link EventLogFormat#MAX_BATCH_SIZE} rather than one per pass. It is committed all the same, since
+ * a pass is a commit point.
*/
- private void sendDiagnosticEvent(DiagnosticEvent diagnosticEvent, boolean isInit) {
- try {
- byte[] data = diagnosticEvent.getJsonValue().toJsonString()
- .getBytes(StandardCharsets.UTF_8);
- handleResponse(eventSender.sendDiagnosticEvent(data, eventsUri));
- if (isInit) {
- // Attempted, not delivered. A failed post gives back an unsuccessful Result rather
- // than throwing, so this marks the init event done either way and the process never
- // retries it. That matches DefaultEventProcessor, which is the behaviour to keep:
- // diagnostics are best-effort telemetry about the SDK, and a retry that outlived
- // its own init would describe a configuration the application has moved on from.
- diagnosticInitSent = true;
- }
- } catch (Exception e) {
- logUnexpectedError(e);
+ private DeliveryPass deliverWaitingThenNew(EventDeliveryRun run) {
+ Set
- * A new event is dropped rather than queued behind the old one. Diagnostics are best-effort
- * telemetry, a post can take tens of seconds against a network that never answers, and queueing
- * would let an outage accumulate events describing an SDK state the application has since moved
- * past -- the same reasoning that makes a failed init event final rather than retried.
+ * Stops at the first failure that may pass, other than a read. The batches behind it would most likely
+ * meet the same network, and the retry takes all of them again.
+ *
+ * @param passedBy the batches whose read failed in this pass, which this adds to
*/
- private boolean claimDiagnosticPost() {
- if (diagnosticPostInFlight.compareAndSet(false, true)) {
- return true;
- }
- logger.debug("Skipped a diagnostic event because the previous one is still being posted");
- return false;
+ private BatchesOutcome deliverPendingBatches(EventDeliveryRun run, Set
+ * Where the directory could not be listed, delivers the batches the store last counted and answers
+ * {@link BatchesOutcome#RETRYABLE}: a batch the listing could not see is still on the disk, and the
+ * listing may work next time.
*
- * @param post what to run there, which must hold a claim from {@link #claimDiagnosticPost()}
+ * @param passedBy the batches whose read failed earlier in this pass, which are not read again, and to
+ * which this adds any whose read fails now
*/
- private void postDiagnostic(Runnable post) {
- Runnable releasing = () -> {
- try {
- post.run();
- } finally {
- diagnosticPostInFlight.set(false);
+ private BatchesOutcome deliverListedBatches(EventDeliveryRun run, Set
+ * Stopped by the backoff instead, it queues itself again for when the wait is up. That wait is one a
+ * post of its own began, such as the second refusal that dropped a batch ahead of the ones it stopped at,
+ * or one a post in the meantime made longer; either way nothing has been tried, and the flushes that
+ * took this retry as theirs, while it was queued, queued no other.
*
- * {@link #updateScheduledTasks} cancels the periodic task when either becomes true, but that is
- * not enough on its own. Canceling does not stop a run already underway, and the init event is
- * submitted before it reaches the executor. Either can arrive here after the state changed.
+ * A pass like the delivery's rather than only the batches it left: the delivery may have closed
+ * nothing, behind a batch that was blocked, and a {@code true} has to mean that the events the
+ * caller was waiting on went too.
*/
- private boolean diagnosticsSuspended() {
- return isStopped() || offline.get() || inBackground.get();
+ private boolean retryPendingBatches() {
+ if (deliveryScheduler.isShuttingDown()) {
+ // close() has queued the release of the store and the sender behind this.
+ return false;
+ }
+ if (offline.get()) {
+ return false;
+ }
+ try {
+ BatchesOutcome outcome = deliverWaitingThenNew(deliveryScheduler.requestedRun(false)).outcome;
+ if (outcome == BatchesOutcome.HELD) {
+ deliveryScheduler.scheduleRetryAgain(this::retryPendingBatches, retryState.remainingWaitNanos());
+ }
+ return outcome == BatchesOutcome.DELIVERED;
+ } catch (RuntimeException e) {
+ logUnexpectedError(e);
+ return false;
+ }
}
- private void handleResponse(EventSender.Result result) {
- if (result == null) {
- return;
+ /**
+ * @return DELIVERED if the batch arrived, or held nothing left to send; FAILED if events were lost,
+ * whether the batch could not be read by this version, or {@link #READ_FAILURES_BEFORE_DROPPING} times
+ * in a row, arrived short, or had its payload refused {@link #BAD_REQUESTS_BEFORE_DROPPING} times in a
+ * row; PASSED_BY if a read failed fewer times than that; RETRYABLE if it is kept to be sent again,
+ * which is every other failure to post it; HELD for a post the backoff did not allow
+ */
+ private BatchesOutcome deliver(EventStore.Batch batch, EventDeliveryRun run) {
+ boolean duringAWait = retryState.remainingWaitNanos() > 0;
+ if (duringAWait && !run.postsThroughTheBackoff()) {
+ // Not an attempt, so it moves nothing in the backoff.
+ return BatchesOutcome.HELD;
+ }
+ EventLogFormat.Payload payload;
+ try {
+ payload = store.payload(batch);
+ } catch (EventStore.UnreadableBatchException e) {
+ // Lost: nothing will ever send it, so it is removed, the rest are still attempted, and the
+ // caller hears that this pass lost events.
+ logger.error("Dropping {} stored event(s) this version cannot read", batch.eventCount);
+ store.remove(batch);
+ reportBatchLost(batch);
+ return BatchesOutcome.FAILED;
+ } catch (EventDirectory.MemoryShortException e) {
+ // The file is intact, and the next pass may find the memory.
+ logger.warn("Could not read stored events, will try again at the next delivery: {}",
+ LogValues.exceptionSummary(e));
+ return BatchesOutcome.PASSED_BY;
+ } catch (IOException e) {
+ if (countReadFailure(batch) >= READ_FAILURES_BEFORE_DROPPING) {
+ logger.error("Dropping {} stored event(s) that could not be read {} times in a row: {}",
+ batch.eventCount, READ_FAILURES_BEFORE_DROPPING, LogValues.exceptionSummary(e));
+ store.remove(batch);
+ reportBatchLost(batch);
+ return BatchesOutcome.FAILED;
+ }
+ // Kept, since a read that failed may not fail next time, but passed by rather than stopped at.
+ // Stopped at, it held back every batch behind it, the open log's included, on every pass for as
+ // long as the read kept failing: for good where expiry cannot open the file to delete it either.
+ logger.warn("Could not read stored events, will try again at the next delivery: {}",
+ LogValues.exceptionSummary(e));
+ return BatchesOutcome.PASSED_BY;
}
- if (result.getTimeFromServer() != null) {
- recordPastTime(result.getTimeFromServer().getTime());
+ endReadFailures(batch);
+ if (payload == null) {
+ // Already delivered by another process of this application, or holding no events. Either way
+ // nothing is lost and there is nothing to keep.
+ store.remove(batch);
+ return BatchesOutcome.DELIVERED;
}
- if (result.isMustShutDown()) {
- disabled = true;
+ // Damage after the first frame: what came before it is sent, and the rest is gone with the file. Counted
+ // once the batch is removed rather than now, since one kept for a retry is read, and found short, again.
+ int leftOut = Math.max(0, batch.eventCount - payload.eventCount);
+ diagnostics.recordEventsInBatch(payload.eventCount);
+ store.startedSending(batch);
+ try {
+ AnalyticsEventSender.Result result = analyticsEventSender.sendBatch(payload, payload.eventCount,
+ batch.payloadId, eventsUri);
+ if (!result.success) {
+ if (duringAWait) {
+ retryState.recordFailureDuringWait(result.unexpected, result.retryAfterMillis);
+ } else {
+ retryState.recordFailure(result.unexpected, result.retryAfterMillis);
+ }
+ if (countBadRequest(batch, result.status) >= BAD_REQUESTS_BEFORE_DROPPING) {
+ // The rest are still attempted, once the backoff allows.
+ logger.error("Dropping {} event(s) whose payload the service refused {} times in a row, "
+ + "last with HTTP {}", batch.eventCount, BAD_REQUESTS_BEFORE_DROPPING, result.status);
+ store.remove(batch);
+ reportBatchLost(batch);
+ return BatchesOutcome.FAILED;
+ }
+ // Kept for any other failure, a 401 included: a valid key is sometimes refused for minutes, and
+ // deleting on a refusal would lose every event recorded meanwhile. The backoff decides when the
+ // retry, the next flush or the next run of the application sends it again.
+ return BatchesOutcome.RETRYABLE;
+ }
+ countBadRequest(batch, result.status);
+ retryState.recordSuccess();
+ if (result.serverTime != null) {
+ committer.recordPastTime(result.serverTime.getTime());
+ }
+ store.remove(batch);
+ if (leftOut > 0) {
+ logger.error("Dropped {} damaged stored event(s) from a batch it delivered", leftOut);
+ lostEvents.addAndGet(leftOut);
+ return BatchesOutcome.FAILED;
+ }
+ return BatchesOutcome.DELIVERED;
+ } catch (Exception e) {
+ logUnexpectedError(e);
+ return BatchesOutcome.FAILED;
+ } finally {
+ // Counted again unless the batch was removed above.
+ store.stoppedSending(batch);
}
}
/**
- * Moves the threshold forwards only. Analytics and diagnostic responses are handled on separate
- * threads, so a plain assignment would let an older reading of the service clock overwrite a
- * newer one and keep debug events alive past the date the service set. A loop rather than
- * {@code accumulateAndGet}, which needs API 24.
+ * @return how many counted reads in a row of this batch have failed, this one included if it is at least
+ * {@link #readFailureSpacingNanos} after the last counted one; once that reaches
+ * {@link #READ_FAILURES_BEFORE_DROPPING} the count is ended, since the batch is dropped
*/
- private void recordPastTime(long timeFromServer) {
- long known;
- do {
- known = lastKnownPastTime.get();
- if (timeFromServer <= known) {
- return;
+ private int countReadFailure(EventStore.Batch batch) {
+ long now = System.nanoTime();
+ synchronized (readFailuresInARow) {
+ ReadFailures failures = readFailuresInARow.get(batch.payloadId);
+ if (failures == null) {
+ failures = new ReadFailures();
+ readFailuresInARow.put(batch.payloadId, failures);
+ } else if (now - failures.lastCountedNanos < readFailureSpacingNanos) {
+ return failures.counted;
+ }
+ failures.counted++;
+ failures.lastCountedNanos = now;
+ if (failures.counted >= READ_FAILURES_BEFORE_DROPPING) {
+ readFailuresInARow.remove(batch.payloadId);
}
- } while (!lastKnownPastTime.compareAndSet(known, timeFromServer));
+ return failures.counted;
+ }
+ }
+
+ /** Ends a batch's run of failed reads, at a read that worked. */
+ private void endReadFailures(EventStore.Batch batch) {
+ synchronized (readFailuresInARow) {
+ readFailuresInARow.remove(batch.payloadId);
+ }
}
/**
- * A debug event is emitted until the date the service gave us passes. We compare against the
- * last date we know to be in the past according to the service as well as the device clock, so
- * that a device whose clock is wrong errs on the side of stopping sooner.
+ * @param status the HTTP status, or 0 where no response arrived
+ * @return how many refusals of the payload in a row this batch has had, counting this response; zero
+ * for any other status, which ends the run, and once the batch is dropped
*/
- private boolean shouldDebugEvent(Long debugEventsUntilDate) {
- if (debugEventsUntilDate == null || debugEventsUntilDate <= 0) {
- return false;
+ private int countBadRequest(EventStore.Batch batch, int status) {
+ synchronized (badRequestLock) {
+ if (status == 0) {
+ // A network or TLS failure leaves the run as it was. The service did not answer, so it said nothing
+ // about the payload: had it ended the run, refusals with network failures between them would never
+ // reach two, and a batch the service will never take would stay the oldest, holding back every
+ // batch behind it for good where nothing expires it, as in memory.
+ return badRequestPayloadId != null && badRequestPayloadId.equals(batch.payloadId)
+ ? badRequestsInARow : 0;
+ }
+ if (!LDUtil.isEventPayloadRefused(status)) {
+ badRequestPayloadId = null;
+ badRequestsInARow = 0;
+ return 0;
+ }
+ if (!batch.payloadId.equals(badRequestPayloadId)) {
+ badRequestPayloadId = batch.payloadId;
+ badRequestsInARow = 0;
+ }
+ badRequestsInARow++;
+ int count = badRequestsInARow;
+ if (count >= BAD_REQUESTS_BEFORE_DROPPING) {
+ badRequestPayloadId = null;
+ badRequestsInARow = 0;
+ }
+ return count;
}
- return debugEventsUntilDate > lastKnownPastTime.get()
- && debugEventsUntilDate > System.currentTimeMillis();
}
/**
- * Must be called holding {@code stateLock}. Once closed, this only ever cancels: close() sets the
+ * Must be called holding {@code stateLock}. Once closed, this only ever cancels. close() sets the
* flag and then calls this under the same lock, so a call that got here first has its tasks
* canceled by close(), and any call after it finds the flag set.
*/
private void updateScheduledTasks(boolean inBackground, boolean offline) {
boolean stopped = closed.get();
// Flushing stays scheduled even while we are offline or in the background; a run while
- // offline returns without doing anything. Canceling it for an outage would restart the
+ // offline commits, where events persist, and sends nothing. Canceling it for an outage would restart the
// interval on every reconnect, and a run of brief outages would then hold events back for
// far longer than one interval. Left running, what an outage buffered goes out at the first
// run after it ends.
- flushTask = enableOrDisableTask(!stopped, flushTask, flushIntervalMillis,
- this::deliverPayload);
- boolean diagnosticsEnabled = !stopped && diagnosticStore != null && !offline && !inBackground;
- diagnosticTask = enableOrDisableTask(diagnosticsEnabled, diagnosticTask,
- diagnosticRecordingIntervalMillis, this::sendDiagnosticStats);
- if (diagnosticsEnabled && !diagnosticInitSent && claimDiagnosticPost()) {
- DiagnosticStore store = diagnosticStore;
- postDiagnostic(() -> {
- // Re-checked on the posting thread: going online and coming to the foreground are
- // two separate calls, and both want to send the init event we never got to send.
- // Suspension is re-checked for the same reason it is for the periodic event, and
- // costs nothing here: getInitEvent consumes nothing, so a skipped init is simply
- // built again the next time diagnostics are enabled.
- if (!diagnosticInitSent && !diagnosticsSuspended()) {
- sendDiagnosticEvent(store.getInitEvent(), true);
- }
- });
+ flushTask = EventUtil.enableOrDisableTask(scheduler, !stopped, flushTask, flushIntervalMillis,
+ guarded(this::flushIntervalEnded));
+ if (!stopped && !offline && !wentOnline) {
+ wentOnlineNanos = System.nanoTime();
+ wentOnline = true;
+ if (!launchGateOpen) {
+ scheduleLaunchGateCheck(TimeUnit.MILLISECONDS.toNanos(pacing.launchDelayMillis));
+ }
}
+ // Events a previous run left on disk have already waited out a whole process, so they do not
+ // wait out an interval as well, only the launch gate. An application that dies within one interval
+ // of every start would otherwise never deliver them.
+ if (!stopped && !offline && launchGateOpen && hasEventsFromPreviousRun.compareAndSet(true, false)) {
+ deliveryScheduler.submit(this::deliverPayload);
+ }
+ diagnostics.setMode(!stopped && !offline && !inBackground, launchGateOpen);
}
- private ScheduledFuture enableOrDisableTask(
- boolean shouldEnable,
- ScheduledFuture currentTask,
- long intervalMillis,
- Runnable task
- ) {
- if (!shouldEnable) {
- if (currentTask != null) {
- currentTask.cancel(false);
- }
- return null;
+ /**
+ * Opens the launch gate once the launch delay has passed since the SDK first went online, and otherwise
+ * looks again then. On the scheduler thread.
+ */
+ private void checkLaunchGate() {
+ if (launchGateOpen || isStopped()) {
+ return;
}
- if (currentTask != null && !currentTask.isDone()) {
- return currentTask;
+ long remaining = TimeUnit.MILLISECONDS.toNanos(pacing.launchDelayMillis) - (System.nanoTime() - wentOnlineNanos);
+ if (remaining > 0) {
+ scheduleLaunchGateCheck(remaining);
+ return;
}
- if (currentTask != null) {
- // Backstop for a throwable that escaped guarded() anyway, such as one thrown while
- // logging the first: the executor marks the repeating future done and never fires it
- // again, and holding that future here would make every later enable a no-op.
- currentTask.cancel(false);
+ launchGateOpen = true;
+ if (hasEventsFromPreviousRun.get()) {
+ logger.debug("Sending events from a previous run, {}ms after going online", pacing.launchDelayMillis);
}
- try {
- // Fixed delay rather than fixed rate: a cached process stops running its tasks without
- // stopping the clock, so at a fixed rate it would come back owing every run it missed and
- // fire them one after another.
- return scheduler.scheduleWithFixedDelay(guarded(task), intervalMillis, intervalMillis,
- TimeUnit.MILLISECONDS);
- } catch (RuntimeException e) { // the executor was shut down under us
- return null;
+ synchronized (stateLock) {
+ updateScheduledTasks(inBackground.get(), offline.get());
}
}
- /**
- * Puts work on the delivery thread, and is the one place that decides whether there is still a
- * thread willing to take it. The {@code isStopped} tests the callers make first are a fast path
- * that saves the work of getting here, not a guarantee; this is what a delivery is actually
- * ordered against close().
- *
- * @return the submitted task, or null if the processor is shutting down or already has
- */
- private Future submit(Runnable task) {
- synchronized (submitLock) {
- if (shuttingDown) {
- // close() has already queued the release of the sender. Work accepted now would be
- // behind it in the queue and would run against a sender that had been closed.
- return null;
- }
- try {
- return scheduler.submit(guarded(task));
- } catch (RuntimeException e) { // the executor was shut down under us
- return null;
- }
- }
+ private void scheduleLaunchGateCheck(long delayNanos) {
+ deliveryScheduler.schedule(this::checkLaunchGate, delayNanos);
}
- /**
- * Keeps an unexpected failure from killing a repeating task or bubbling out of the executor.
- * Anything that escapes a run suppresses the rest of a {@code scheduleWithFixedDelay} series,
- * so this catches {@code Throwable} and not just {@code Exception}: a
- * {@code StackOverflowError} from nested {@code LDValue} data or an {@code OutOfMemoryError}
- * growing the payload stream would otherwise stop flushing for the life of the process with
- * nothing logged.
- */
+ /** As {@link EventUtil#guarded}. */
private Runnable guarded(Runnable task) {
- return () -> {
- try {
- task.run();
- } catch (Throwable t) {
- logUnexpectedError(t);
- }
- };
+ return EventUtil.guarded(logger, task);
}
private boolean isStopped() {
- return closed.get() || disabled;
+ return closed.get();
}
private void logUnexpectedError(Throwable e) {
- logger.error("Unexpected error in event processor: {}", LogValues.exceptionSummary(e));
- logger.debug("{}", LogValues.exceptionTrace(e));
+ EventUtil.logUnexpectedError(logger, e);
}
}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiskEventLog.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiskEventLog.java
new file mode 100644
index 000000000..2762f4210
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiskEventLog.java
@@ -0,0 +1,289 @@
+package com.launchdarkly.sdk.android;
+
+import com.launchdarkly.logging.LDLogger;
+import com.launchdarkly.logging.LogValues;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * The file of events on disk this process appends to, not to be confused with diagnostic logging.
+ *
+ * One per store rather than one per directory, so that only one writer ever appends to a given log. Each is
+ * named, by {@link EventDirectory} as every file there is, for a UUID picked when it is opened, which is what
+ * keeps two writers apart: nothing about it comes from
+ * the application, and no two processes, or two stores in one, ever pick the same one.
+ *
+ * The owner holds an exclusive lock on its log for as long as it is open, and the kernel releases it when the
+ * descriptor closes, including when the process dies. A log no one holds the lock on therefore belongs to no
+ * one alive, whatever process wrote it, and recovery takes it. A log is locked before it takes the name
+ * recovery looks for, so that a log being opened is never taken for one whose owner is gone.
+ *
+ * A POSIX lock belongs to the process rather than to the descriptor it was taken through, and closing any
+ * descriptor this process has on the file releases it. So nothing in this process opens another store's log:
+ * the logs open here are remembered, and recognized as live without a descriptor being opened on them.
+ *
+ * Belongs to an {@link EventStore}, which makes every call holding its {@code ioLock}. The lock is handed
+ * in so that each call can check it is held, rather than this taking a lock of its own that would have
+ * to be ordered against the store's.
+ */
+final class DiskEventLog {
+ /** How the log is opened, so a test can make a write fail partway through a session, as a full disk does. */
+ interface Opener {
+ FileOutputStream openForAppending(File file) throws IOException;
+ }
+
+ /** The absolute paths of the logs open in this process, which nothing here may open a descriptor on. */
+ private static final Set
+ * A log open in this process is answered from memory. Any other is answered by trying for a shared lock
+ * on it, which fails while its owner holds the exclusive one. A log that cannot be looked at counts as
+ * held: leaving a dead process's log for a later run costs nothing, while taking a live one's would lose
+ * what it appends next.
+ */
+ static boolean isHeldByLiveProcess(File log) {
+ synchronized (openInThisProcess) {
+ if (openInThisProcess.contains(log.getAbsolutePath())) {
+ return true;
+ }
+ }
+ try (RandomAccessFile probe = new RandomAccessFile(log, "r");
+ FileChannel channel = probe.getChannel();
+ FileLock held = channel.tryLock(0L, Long.MAX_VALUE, true)) {
+ return held == null;
+ } catch (FileNotFoundException e) {
+ // Gone, so no one is appending to it.
+ return false;
+ } catch (IOException | OverlappingFileLockException e) {
+ return true;
+ }
+ }
+
+ /** Where the log is, whether it exists yet or not. */
+ File file() {
+ return file;
+ }
+
+ /** Requires {@code ioLock}. */
+ boolean exists() {
+ assert Thread.holdsLock(ioLock);
+ return file.exists();
+ }
+
+ /**
+ * Whether this store has the log open and locked, so that no other process can have recovered it, nor
+ * any other store in this one. Requires {@code ioLock}.
+ */
+ boolean isHeld() {
+ assert Thread.holdsLock(ioLock);
+ return output != null && lock != null;
+ }
+
+ /**
+ * Appends to the log, opening it first where it is not open.
+ *
+ * Requires {@code ioLock}.
+ *
+ * @throws IOException if the log could not be opened or written to; none of {@code bytes} is then left
+ * in the log, and the log is closed, so the next append starts a new one
+ */
+ void append(byte[] bytes) throws IOException {
+ assert Thread.holdsLock(ioLock);
+ FileOutputStream stream = outputForAppending();
+ long start = -1;
+ try {
+ start = stream.getChannel().size();
+ stream.write(bytes);
+ } catch (IOException | RuntimeException e) {
+ // Writing that fails partway can leave whole frames behind. The caller keeps every one of
+ // the bytes in memory to deliver from there, so frames left here as well would be sent
+ // twice: beside the memory copy when the log is closed this session, or by recovery on the
+ // next launch.
+ if (start >= 0 && !truncate(stream, start) && start <= EventLogFormat.FILE_HEADER_SIZE) {
+ // A log with no events before this write is never closed into a batch, and a later run
+ // recovering it would send what landed under no count that leaves it out. Memory holds
+ // all of it.
+ if (!file.delete() && file.exists()) {
+ logger.warn("Could not remove an event log holding only a failed write");
+ }
+ }
+ closeOutput();
+ throw e;
+ }
+ }
+
+ /**
+ * Truncates the log to {@code length}, where a failed write left it longer. Where it cannot, the
+ * batch this log is closed into still delivers only the events it was closed holding, which leaves out
+ * the frames left behind.
+ *
+ * @return false if the log could not be truncated
+ */
+ private boolean truncate(FileOutputStream stream, long length) {
+ try {
+ stream.getChannel().truncate(length);
+ return true;
+ } catch (IOException | RuntimeException e) {
+ logger.warn("Could not truncate a failed write to the event log: {}",
+ LogValues.exceptionSummary(e));
+ return false;
+ }
+ }
+
+ /**
+ * Moves the log to {@code destination}, so that the next append starts a new one.
+ *
+ * Closed after the move rather than before it, so that the log is still locked while it moves: unlocked
+ * under this name, it could be taken by another process recovering logs in between.
+ *
+ * A move that fails leaves a log that is still there open and locked, to be appended to and moved
+ * again. Closed, it would be no longer this store's to write, and the next append would start another
+ * log while the caller still counted this one's events as the open log's: they would be out of reach
+ * this session, and the next batch would be named for more events than it holds.
+ *
+ * Requires {@code ioLock}.
+ *
+ * @return whether the log was moved
+ */
+ boolean closeMovingTo(File destination) {
+ assert Thread.holdsLock(ioLock);
+ boolean moved = file.renameTo(destination);
+ if (moved || !file.exists()) {
+ closeOutput();
+ }
+ return moved;
+ }
+
+ /**
+ * Closes the stream, leaving the log where it is. Requires {@code ioLock}.
+ *
+ * Unlocked, it is from here on a log like one a process that died left behind, which recovery may take
+ * at any moment. So this log is never opened again: the next append starts a new one.
+ */
+ void closeOutput() {
+ assert Thread.holdsLock(ioLock);
+ if (output == null) {
+ return;
+ }
+ try {
+ output.close();
+ } catch (IOException e) {
+ logger.debug("Could not close the event log: {}", LogValues.exceptionSummary(e));
+ }
+ output = null;
+ lock = null;
+ forget(file);
+ }
+
+ /**
+ * Takes the lock on the open log again, after something in this process closed a descriptor of its own
+ * on the file, which released it. Requires {@code ioLock}.
+ */
+ void relockAfterReading() {
+ assert Thread.holdsLock(ioLock);
+ if (output == null) {
+ return;
+ }
+ try {
+ if (lock != null) {
+ lock.release();
+ }
+ lock = output.getChannel().tryLock();
+ } catch (IOException | RuntimeException e) {
+ lock = null;
+ }
+ if (lock == null) {
+ logger.debug("Could not lock the event log again after reading it");
+ }
+ }
+
+ /**
+ * Opens a new log under a name of its own: created under {@link EventDirectory#OPENING_LOG_PREFIX},
+ * locked, given its header, and only then moved to the name recovery looks for.
+ *
+ * Requires {@code ioLock}.
+ */
+ private FileOutputStream outputForAppending() throws IOException {
+ if (output != null) {
+ return output;
+ }
+ if (!directory.exists() && !directory.mkdirs() && !directory.isDirectory()) {
+ throw new IOException("could not create " + directory);
+ }
+ // Never the name of an earlier open in this session, whatever exists() says of it: that log is unlocked
+ // and no longer this store's to write, and exists() answers false where it cannot look, so the rename
+ // below would replace it and lose its events.
+ file = EventDirectory.newLogFile(directory);
+ File opening = EventDirectory.openingFileFor(file);
+ remember(file);
+ FileOutputStream stream = null;
+ try {
+ stream = opener.openForAppending(opening);
+ FileLock taken = stream.getChannel().tryLock();
+ if (taken == null) {
+ throw new IOException("could not lock " + opening);
+ }
+ stream.write(EventLogFormat.fileHeader());
+ if (!opening.renameTo(file)) {
+ throw new IOException("could not move " + opening + " to " + file);
+ }
+ lock = taken;
+ output = stream;
+ return output;
+ } catch (IOException | RuntimeException e) {
+ // Not yet in output, so closeOutput would not reach it.
+ if (stream != null) {
+ try {
+ stream.close();
+ } catch (IOException ignored) {
+ // The open already failed; that is the error worth reporting.
+ }
+ }
+ if (opening.exists() && !opening.delete()) {
+ logger.debug("Could not delete {}", opening);
+ }
+ forget(file);
+ throw e;
+ }
+ }
+
+ private static void remember(File log) {
+ synchronized (openInThisProcess) {
+ openInThisProcess.add(log.getAbsolutePath());
+ }
+ }
+
+ private static void forget(File log) {
+ synchronized (openInThisProcess) {
+ openInThisProcess.remove(log.getAbsolutePath());
+ }
+ }
+}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventCommitter.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventCommitter.java
new file mode 100644
index 000000000..9534e3958
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventCommitter.java
@@ -0,0 +1,818 @@
+package com.launchdarkly.sdk.android;
+
+import com.launchdarkly.logging.LDLogger;
+import com.launchdarkly.sdk.EvaluationReason;
+import com.launchdarkly.sdk.LDContext;
+import com.launchdarkly.sdk.LDValue;
+import com.launchdarkly.sdk.internal.events.Event;
+import com.launchdarkly.sdk.internal.events.EventSummarizer;
+import com.launchdarkly.sdk.internal.events.Sampler;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Where {@link DirectEventProcessor} accepts events, holds them and commits them to the store.
+ *
+ * Recording an event summarizes it immediately and, only if it has to be delivered in full, holds it. A full
+ * event is kept as an {@link Event} until a commit turns the run of them into bytes and writes them. So
+ * evaluations of untracked flags cost a counter increment, and evaluations of tracked ones cost that plus a
+ * list append. Nothing is encoded on the thread that evaluated.
+ *
+ * Where the line falls is the subject of O12 in the event-encoding research, and it falls between the events
+ * an application asked for by name and the ones it did not. At {@code IMMEDIATE}, {@code track} and
+ * {@code identify} write themselves before returning, because an application reporting an error is saying
+ * this matters more than the microseconds it costs, and the crash it describes may be moments away. They
+ * encode and write only themselves: what is held, and the summaries, which can take megabytes to encode, are
+ * left to a commit queued on the commit executor in the same call, so the exposures leading up to the error
+ * follow it moments later rather than costing the caller's thread. Everything else is committed once
+ * {@link #PENDING_COMMIT_THRESHOLD} events have accumulated where persistence is on, on {@code flush}, at a
+ * delivery, on backgrounding, or when the application flushes from its own crash handler. Backgrounding
+ * commits through {@code flush}: {@link ConnectivityManager} calls it on every move to the background, and
+ * {@link #commitBeforeDelivery} writes on that thread wherever events persist. {@code setInBackground} only
+ * reschedules the processor's tasks.
+ *
+ * Which of the two {@code track} gets is the application's choice, through
+ * {@link com.launchdarkly.sdk.android.integrations.EventProcessorBuilder#eventPersistence}. Only at
+ * {@code IMMEDIATE} can {@code track} promise the event is on disk by the time it returns; at
+ * {@code DEFERRED} it costs what an evaluation does.
+ *
+ * Threads. Recording runs on whichever thread evaluated or tracked. Commits run on {@link #commitExecutor},
+ * or on the caller's thread for a flush, one at a time under {@link #commitLock}. A delivery commits too, on
+ * the event processor's delivery thread, before it closes a batch. A {@code track} or {@code identify} at
+ * {@code IMMEDIATE} writes its one event without {@link #commitLock}, so that it never waits for a commit
+ * encoding a run.
+ */
+final class EventCommitter {
+ /**
+ * How many full events may be held unencoded before one of them pays for a commit.
+ *
+ * It counts events rather than bytes because there are no bytes until the commit runs. Each commit writes
+ * everything it encoded at once, so this, with the commit points, is what decides how large each write to
+ * disk is. It sets two things at once.
+ *
+ * The first is how many evaluations a termination can take. That is narrower than everything a crash
+ * could take. An uncaught exception runs the application's handler while the process is still alive, and
+ * a handler that flushes commits the whole held run. So this is the window for the terminations that run
+ * nothing on the way out -- {@code SIGKILL}, a native crash, an ANR kill, and the system reclaiming a
+ * backgrounded process. A {@code track} at {@code IMMEDIATE} narrows it too, because it queues a commit.
+ *
+ * The second is what a {@code flush} usually costs, since it encodes whatever is held before returning.
+ * The commit executor keeps the run drained, so the common case is far below this; raising it trades that
+ * cost against the number of writes.
+ *
+ * Neither is a ceiling. Reaching this only schedules a commit, and recording does not wait for it, so
+ * where events arrive faster than the commit executor encodes them, or the executor is starved, the run
+ * grows up to the event capacity, and the next flush encodes all of it.
+ */
+ static final int PENDING_COMMIT_THRESHOLD = 32;
+
+ // Set in the constructor and never changed.
+
+ private final EventStore store;
+ /**
+ * Where a commit no caller waits for runs: a thread of its own, not the event processor's delivery thread.
+ *
+ * That thread can be held by one delivery for as long as two network timeouts. A commit queued behind
+ * that would leave the pending run in memory for all of it, and let it fill toward capacity meanwhile.
+ */
+ private final ExecutorService commitExecutor;
+ /**
+ * How many events the SDK will hold in total, across {@link #pending} and the store.
+ */
+ private final int capacity;
+ /**
+ * Whether {@code track} and {@code identify} write before returning, and a flush writes even where
+ * nothing persists, rather than leaving that work to a commit off the caller's thread.
+ */
+ private final boolean commitOnCallerThread;
+ /** The event processor's: set once it closes, after which nothing more is recorded. */
+ private final AtomicBoolean closed;
+ /** The event processor's count of events accepted and then lost for good, which flushes answer for. */
+ private final AtomicLong lostEvents;
+ /** The event processor's count of events dropped for capacity, which diagnostics carry. */
+ private final AtomicLong droppedEvents;
+ private final LDLogger logger;
+
+ // Guarded by commitLock.
+
+ /**
+ * Held for the whole of a commit, so that only one runs at a time.
+ *
+ * This is what a flush's guarantee rests on. Without it a commit already in flight could take the
+ * caller's events out of {@link #pending} before the caller got there, leaving the caller nothing to
+ * write and returning while those bytes were still being produced somewhere else. Waiting here instead
+ * means that when the call returns the events are on disk, whichever commit put them there.
+ *
+ * Distinct from {@link #recordLock}, which is only held long enough to hand the run and the counters
+ * over: encoding under this lock must not block a thread that is merely recording.
+ */
+ private final Object commitLock = new Object();
+
+ /**
+ * The last recording a finished commit covered. Written under {@link #commitLock}, and read outside it
+ * so that a caller whose recording is already durable does not queue for the lock at all.
+ *
+ * This is what makes concurrent flushes a group commit. The first caller's
+ * commit takes every recording made before it took the run, including those of callers now waiting
+ * behind it. They find their recordings covered and return, and the next caller still uncovered
+ * commits everything that arrived in the meantime. Without it each waiter committed again in turn,
+ * and each of those commits wrote a summary of its own.
+ */
+ private volatile long committedSequence;
+
+ // Guarded by recordLock.
+
+ /**
+ * Guards everything one recording writes: {@link #pending} and the summary counters behind
+ * {@link #eventBuffer}.
+ *
+ * One evaluation can produce a counter, a full event and a debug event, and the three have to
+ * land together. Taken separately, a flush landing between them splits one evaluation across two
+ * payloads. A {@code close()} landing between them delivers the counter and then refuses the
+ * full event, leaving a summary that says an evaluation happened and no event to go with it.
+ *
+ * Held for the appends and the handover of the run, never across the encoding. Recording runs on
+ * whichever thread evaluated a flag, which on Android is usually the main one. The worst it may
+ * wait for is another thread's memory operation; if the encoder ran under this lock, an
+ * evaluation would instead wait on the dominant cost of the whole path.
+ */
+ private final Object recordLock = new Object();
+
+ /**
+ * Full events recorded but not yet encoded, guarded by {@link #recordLock}.
+ *
+ * Held rather than serialized because serializing early would not make them durable: the store stages
+ * bytes into memory too, and only {@link EventStore#commit()} reaches the file. Both forms are equally
+ * lost to a crash, so the encoding may as well happen where it is cheapest.
+ */
+ private final List
+ * Taken there rather than left, so the summarizer's contexts are cleared at each of them as they were when
+ * they committed the counters themselves: an application identifying its way through more contexts than
+ * capacity faster than the commit executor keeps up would otherwise find the cardinality limit binding.
+ * Every commit takes them along with {@link #pending}, so an evaluation's counter and its full event are
+ * still staged together.
+ */
+ private final List
+ * At {@code DEFERRED} it writes nothing itself (durability §13.3), and is committed with what is held
+ * at the pending threshold, a flush, a delivery, backgrounding or close. A commit per call would cost
+ * an application that tracks every few seconds one encode and write per event, each taking
+ * {@link #recordLock} from the threads evaluating flags and the store's locks from delivery.
+ */
+ void recordAtCommitPoint(Event event) {
+ if (commitOnCallerThread && store.isPersisting()) {
+ writeAlone(event);
+ } else {
+ record(event);
+ }
+ }
+
+ /**
+ * Encodes and writes one event on the caller's thread, and queues a commit of everything else.
+ *
+ * Only the event itself, so the caller pays for one small encode and one write however much is held:
+ * a commit encodes the held run and every summary, and a summary with a large context or many flags
+ * takes megabytes, which is a frozen frame on the main thread. They follow on the commit executor,
+ * queued here. The event is written ahead of the run held before it, which LaunchDarkly does not mind:
+ * every event carries its own creation date.
+ *
+ * Not under {@link #commitLock}, which a commit holds for the whole of its encoding, so this never waits
+ * for one. The store writes the one frame rather than committing, which would also write whatever a
+ * commit in flight has staged so far, and the encoder shares nothing between threads; the one wait left
+ * is for the store's write lock, held by a write or by a delivery reading a batch back.
+ */
+ private void writeAlone(Event event) {
+ try {
+ boolean admitted;
+ synchronized (recordLock) {
+ // As in record: orders this against close()'s final commit, which this does not join, so an
+ // event admitted here is written before the store is released, and one refused never is.
+ if (closed.get()) {
+ return;
+ }
+ admitted = admit(event);
+ if (admitted) {
+ // Reserved where it is admitted, as a run is where it leaves pending, so capacity counts
+ // it while it is encoded and a recording on another thread cannot take its place.
+ store.reserve(1);
+ }
+ takenSummaries.addAll(eventBuffer.takeSummaries());
+ summaryContextsExceeded.set(false);
+ }
+ if (admitted) {
+ byte[] serialized = eventBuffer.serialize(event);
+ if (serialized == null) {
+ // Counted by the commit queued below, which takes the failure.
+ store.releaseReservations(1);
+ } else if (!store.writeReservedAlone(serialized)) {
+ // Only the bytes are checked: a frame or byte capacity it turns out not to fit.
+ reportRefusedByStore("event", serialized);
+ }
+ }
+ scheduleCommit();
+ } catch (RuntimeException e) {
+ // As in recordEvaluation: on the caller's thread, so a failure is logged, not thrown.
+ logUnexpectedError(e);
+ }
+ }
+
+ /**
+ * Holds an event for the next commit to encode, counting it as dropped if the SDK is already full.
+ *
+ * Capacity is consulted before anything is encoded, so an event that will not be kept is never encoded.
+ * That ordering is what bounds an application re-evaluating a tracked flag in a render loop: once the
+ * limit is reached the cost of an evaluation falls back to its summary counter, however fast the loop runs.
+ *
+ * @return the recording's sequence number, for the commit point that follows it; 0 if it was refused
+ */
+ long record(Event event) {
+ try {
+ long sequence;
+ boolean needsCommit;
+ synchronized (recordLock) {
+ // The close check that decides the outcome, as against the fast path the public record methods
+ // take before building the event. The commit lifts the run out under this same lock, so testing
+ // the flag here orders a record against close()'s final commit: either the event is in the list
+ // before that commit takes it, or it is refused. Tested outside the lock the two interleave, and
+ // an event can be left in a list that nothing will drain again.
+ if (closed.get()) {
+ return 0;
+ }
+ sequence = ++recordedSequence;
+ addPending(event);
+ needsCommit = pending.size() >= PENDING_COMMIT_THRESHOLD;
+ }
+ if (needsCommit) {
+ scheduleCommitWherePersisting();
+ }
+ return sequence;
+ } catch (RuntimeException e) {
+ // As in recordEvaluation: on the caller's thread, so a failure is logged, not thrown.
+ logUnexpectedError(e);
+ return 0;
+ }
+ }
+
+ /**
+ * Commits unless a finished commit already covers recording {@code through}, checked again once the lock
+ * is held, since the commit a caller waited behind is the one most likely to have covered it.
+ */
+ private void commitDurablyThrough(long through) {
+ if (committedSequence >= through) {
+ return;
+ }
+ synchronized (commitLock) {
+ if (committedSequence >= through) {
+ return;
+ }
+ commitDurably();
+ }
+ }
+
+ /** @return the sequence number of the last recording so far */
+ private long lastRecordedSequence() {
+ synchronized (recordLock) {
+ return recordedSequence;
+ }
+ }
+
+ /**
+ * Queues the commit a full pending run asks for, if there is a disk for it to reach.
+ *
+ * Without persistence a commit makes nothing durable. All it would do is move the encoding into the
+ * middle of the application's evaluations, competing with them for the CPU and splitting their
+ * counters across summaries. Left alone, the run waits for the flush, which encodes it anyway, and
+ * capacity still bounds how much is held.
+ */
+ private void scheduleCommitWherePersisting() {
+ if (store.isPersisting()) {
+ scheduleCommit();
+ }
+ }
+
+ /** Queues a commit unless one is already queued, so a run of recordings asks for one rather than many. */
+ private void scheduleCommit() {
+ // Read before the compare-and-set, which takes the line for writing even when it fails, and fails for
+ // every event recorded past the threshold while a commit is already queued.
+ if (!commitScheduled.get() && commitScheduled.compareAndSet(false, true)) {
+ try {
+ commitExecutor.execute(guarded(this::runScheduledCommit));
+ } catch (RuntimeException e) { // shut down by close(), whose own commit takes these events
+ commitScheduled.set(false);
+ }
+ }
+ }
+
+ /**
+ * Clears the scheduling flag before committing, not after, so events recorded while this runs can queue
+ * a commit of their own rather than waiting for the next one to be triggered.
+ *
+ * Does nothing once closed. {@link DirectEventProcessor#close()} sets the flag before its final commit,
+ * which takes everything recorded up to then, and nothing is recorded after it. A commit reaching here
+ * later could only run against a store that has been released.
+ */
+ private void runScheduledCommit() {
+ commitScheduled.set(false);
+ if (closed.get()) {
+ return;
+ }
+ commitDurably();
+ }
+
+ /**
+ * The commit point {@link DirectEventProcessor#flush()} and {@link DirectEventProcessor#blockingFlush()}
+ * start with, before anything is handed to the events thread.
+ *
+ * Written on the caller's thread wherever events persist, not only where the application asked for
+ * that as it does for {@code track} and {@code identify}. A flush is what an application calls when it
+ * is about to end the process on purpose, and the process may end before a commit it only queued
+ * has run. Where nothing persists, a commit makes nothing durable, and is left to the delivery.
+ *
+ * It does not wait for the events thread, which can be held by a post or by a retry long after the
+ * caller has moved on. The delivery commits again when it runs, but that is too late to be the only
+ * commit.
+ */
+ void commitBeforeDelivery() {
+ try {
+ if (commitOnCallerThread || store.isPersisting()) {
+ commitDurablyThrough(lastRecordedSequence());
+ }
+ } catch (RuntimeException e) {
+ // As in recordEvaluation: every commit point is on the caller's thread.
+ logUnexpectedError(e);
+ }
+ }
+
+ /**
+ * The commit point {@link DirectEventProcessor#flushAsync()} starts with: the same commit as
+ * {@link #commitBeforeDelivery}, queued on the commit executor rather than written on the caller's thread.
+ *
+ * A caller of {@code flushAsync} has a deadline, which is how {@code flushAndWait} bounds the whole call,
+ * and it can only enforce one once it holds the future. Each step of a commit can block for as long as
+ * the device takes: on {@link #commitLock} behind a commit or a delivery that stalled on the disk, on the
+ * store's I/O lock, and on its own writing. Committed here, any of them ran before the deadline could
+ * apply, so a stalled flash write outlasted the timeout of the crash handler on its way out.
+ *
+ * Queued rather than left to the delivery, because the events thread can be held by a post or by a
+ * retry long after the caller has stopped waiting, and {@code flushAndWait} is for a caller that is about
+ * to be killed. The commit executor is not held by the network. The future still answers for the write:
+ * the delivery commits again before it sends, so a {@code true} means the events were written first. A
+ * commit that outlasts the caller's wait is left to finish, as {@code close()} leaves its own.
+ */
+ void queueCommitBeforeDelivery() {
+ try {
+ if (commitOnCallerThread || store.isPersisting()) {
+ long through = lastRecordedSequence();
+ commitExecutor.execute(guarded(() -> {
+ // As in runScheduledCommit: close() writes everything recorded before it with its own
+ // commit, and a commit reaching here later could only run against a released store.
+ if (!closed.get()) {
+ commitDurablyThrough(through);
+ }
+ }));
+ }
+ } catch (RuntimeException e) {
+ // Shut down by close(), whose own commit takes these events; or as in recordEvaluation,
+ // a failure on the caller's thread is logged rather than thrown.
+ if (!closed.get()) {
+ logUnexpectedError(e);
+ }
+ }
+ }
+
+ /**
+ * Encodes everything recorded since the last commit and makes it survive the process.
+ *
+ * This is the only place events are serialized. It runs on the commit executor when the pending run
+ * reaches {@link #PENDING_COMMIT_THRESHOLD}, and on the caller's thread for {@code flush}, where the
+ * caller has asked to wait for exactly this.
+ *
+ * The evaluations counted since the last commit are written too, because a summary is only worth having
+ * if it covers the evaluations that led up to whatever is about to happen.
+ */
+ void commitDurably() {
+ synchronized (commitLock) {
+ List
+ * Both under {@link #commitLock}, for two reasons. A commit landing between the two would stage
+ * events into the log this batch is about to close. That splits what one commit staged across
+ * two payloads: an evaluation's counter in this batch and its event in the next. And the mark
+ * has to be the count as of the close, because that is what this delivery can answer for: a loss
+ * after it is one whose events were never in this batch.
+ *
+ * A flush, requested or periodic, has usually just committed before it delivers, so this commit often
+ * finds nothing to do. It cannot be the one left out: that earlier commit is only where events persist,
+ * and the retry and the final delivery at close reach here without it.
+ *
+ * @return the loss count as of the close, and whether the close took every committed event into a
+ * batch; both under the lock, so they cover this close and nothing a later commit does
+ */
+ Close commitAndCloseBatch() {
+ synchronized (commitLock) {
+ commitDurably();
+ store.closeBatch();
+ reportVanished(store.takeVanishedLogEvents());
+ return new Close(lostEvents.get(), !store.holdsUnclosedEvents());
+ }
+ }
+
+ /**
+ * {@link #commitAndCloseBatch()} for a pass that found an earlier batch blocked, and so closes nothing.
+ * The mark is as of this commit, the last point anything the caller recorded was staged.
+ *
+ * @return the loss count as of the commit
+ */
+ long commitLeavingTheLogOpen() {
+ synchronized (commitLock) {
+ commitDurably();
+ // A commit that fills the log closes it, and can find it gone.
+ reportVanished(store.takeVanishedLogEvents());
+ return lostEvents.get();
+ }
+ }
+
+ /** Counts as lost the events of an open log a close found gone. */
+ private void reportVanished(int events) {
+ if (events <= 0) {
+ return;
+ }
+ logger.error("{} stored event(s) were deleted before they could be sent", events);
+ lostEvents.addAndGet(events);
+ }
+
+ /**
+ * Commits everything recorded so far for close(), waiting for it no longer than the budget allows.
+ *
+ * On the commit executor rather than the caller's thread, because each step of a commit can block
+ * for as long as the device takes. {@link #commitLock} may be held by a commit or a delivery that
+ * stalled on the disk, and the store's I/O lock by the disk I/O it stalled on. This commit's own
+ * writing can stall the same way. Committing on the caller's thread was bounded by none of them, so
+ * a stalled flash write turned into an ANR on the main thread.
+ *
+ * A commit that outlasts the budget is left to finish, for the same reason as the delivery. The
+ * executor thread is a daemon, and returning from close() does not end the process. Writing cut off
+ * partway leaves a torn frame that the next run reads past, not events lost.
+ *
+ * @param budgetMillis the whole of close()'s budget, for the warning
+ * @return the commit, for the store's release to wait on, or null if it could not be queued
+ */
+ Future commitWithinBudget(long deadlineNanos, long budgetMillis) {
+ Future commit;
+ try {
+ commit = commitExecutor.submit(guarded(this::commitDurably));
+ } catch (RuntimeException e) { // shut down under us; the final delivery still commits first
+ return null;
+ }
+ try {
+ commit.get(Math.max(0, deadlineNanos - System.nanoTime()), TimeUnit.NANOSECONDS);
+ } catch (TimeoutException e) {
+ logger.warn("Gave up waiting for the final event commit after {}ms;" +
+ " it continues in the background", budgetMillis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } catch (ExecutionException e) {
+ logUnexpectedError(e.getCause() == null ? e : e.getCause());
+ }
+ return commit;
+ }
+
+ /** Stops the commit thread taking more. Commits already queued there still run, and find the processor closed. */
+ void shutdown() {
+ commitExecutor.shutdown();
+ }
+
+ /**
+ * Encodes a run of events as one run and stages the bytes.
+ *
+ * The run was taken under {@link #recordLock} and is encoded outside it, so recording does not wait on
+ * the encoder. Staging bypasses capacity because the decision to keep these events was already made in
+ * {@link #record}, and refusing them here would drop events the SDK has counted as accepted. The
+ * reservation taken with the run is what keeps them counted until they are staged. The byte capacity
+ * can still refuse one, since only its encoded size says whether it fits, and an event refused then is
+ * lost rather than dropped.
+ *
+ * Requires {@link #commitLock}: two threads staging separate runs would stage them in whichever order
+ * they finished encoding, which is not the order they were recorded in.
+ */
+ private void stageRun(List
+ * A refused evaluation is a loss in the same sense a refused event is -- nothing later reconstructs
+ * a counter -- so it is reported the same way, through the dropped count diagnostics carry.
+ */
+ private void reportContextsExceeded(boolean warn) {
+ // Warned once per delivery run rather than once per process: the summarizer's contexts are
+ // cleared with each run, so a later overflow is a new one worth hearing about.
+ if (warn) {
+ logger.warn("Exceeded the number of contexts that can be summarized at once." +
+ " Increase capacity to avoid dropping evaluations.");
+ }
+ droppedEvents.incrementAndGet();
+ }
+
+ /**
+ * Moves the threshold forwards only. Analytics and diagnostic responses are handled on separate
+ * threads, so a plain assignment would let an older reading of the service clock overwrite a
+ * newer one and keep debug events alive past the date the service set. A loop rather than
+ * {@code accumulateAndGet}, which needs API 24.
+ */
+ void recordPastTime(long timeFromServer) {
+ long known;
+ do {
+ known = lastKnownPastTime.get();
+ if (timeFromServer <= known) {
+ return;
+ }
+ } while (!lastKnownPastTime.compareAndSet(known, timeFromServer));
+ }
+
+ /**
+ * A debug event is emitted until the date the service gave us passes. We compare against the
+ * last date we know to be in the past according to the service as well as the device clock, so
+ * that a device whose clock is wrong errs on the side of stopping sooner.
+ */
+ private boolean shouldDebugEvent(Long debugEventsUntilDate) {
+ if (debugEventsUntilDate == null || debugEventsUntilDate <= 0) {
+ return false;
+ }
+ return debugEventsUntilDate > lastKnownPastTime.get()
+ && debugEventsUntilDate > System.currentTimeMillis();
+ }
+
+ private Runnable guarded(Runnable task) {
+ return EventUtil.guarded(logger, task);
+ }
+
+ private void logUnexpectedError(Throwable e) {
+ EventUtil.logUnexpectedError(logger, e);
+ }
+}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDeliveryRun.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDeliveryRun.java
new file mode 100644
index 000000000..e4aa41e88
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDeliveryRun.java
@@ -0,0 +1,84 @@
+package com.launchdarkly.sdk.android;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * One delivery's pass over the batches: whether it may post while the backoff would not allow it, and, for one
+ * the processor started on its own, the gap it keeps between batches.
+ *
+ * Made by {@link EventDeliveryScheduler} as the pass begins, and only used on the scheduler thread that runs it.
+ */
+final class EventDeliveryRun {
+ /** Whether something has happened that ends a paced run's wait for good: close, going offline, a flush. */
+ interface PacingEnd {
+ boolean reached();
+ }
+
+ private final boolean throughTheBackoff;
+ private final long gapNanos;
+ /** What {@link #awaitTurn()} waits on, which {@link EventDeliveryScheduler#wakePacedDelivery()} notifies. */
+ private final Object turnLock;
+ private final PacingEnd pacingEnd;
+
+ private boolean paced;
+ private int posts;
+ private long lastPostEndedNanos;
+
+ EventDeliveryRun(boolean throughTheBackoff, boolean paced, long gapMillis, Object turnLock, PacingEnd pacingEnd) {
+ this.throughTheBackoff = throughTheBackoff;
+ this.paced = paced;
+ this.gapNanos = TimeUnit.MILLISECONDS.toNanos(gapMillis);
+ this.turnLock = turnLock;
+ this.pacingEnd = pacingEnd;
+ }
+
+ /**
+ * Set for close()'s final delivery and a flush bypassing the backoff, which may post while the backoff
+ * would not allow it.
+ */
+ boolean postsThroughTheBackoff() {
+ return throughTheBackoff;
+ }
+
+ /**
+ * Holds the next batch of a paced run until a gap has passed since the last one answered. A backlog sent
+ * unasked otherwise takes the network for as long as it lasts, an {@code identify}'s flag request included.
+ * The first batch of a run does not wait.
+ *
+ * A delivery someone asked for, or close, ends the pacing, and the rest go out back to back: the delivery
+ * either queues runs on this thread, behind this one. Going offline ends the wait too, but the run then
+ * stops before its next post, leaving the rest for a later delivery.
+ */
+ void awaitTurn() {
+ if (!paced || posts == 0) {
+ return;
+ }
+ long gapEnds = lastPostEndedNanos + gapNanos;
+ try {
+ synchronized (turnLock) {
+ while (true) {
+ if (pacingEnd.reached()) {
+ paced = false;
+ return;
+ }
+ long remaining = gapEnds - System.nanoTime();
+ if (remaining <= 0) {
+ return;
+ }
+ TimeUnit.NANOSECONDS.timedWait(turnLock, remaining);
+ }
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ paced = false;
+ }
+ }
+
+ /** Marks a post as answered, which is when the gap before a paced run's next one starts. */
+ void postEnded() {
+ if (paced) {
+ posts++;
+ lastPostEndedNanos = System.nanoTime();
+ }
+ }
+}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDeliveryScheduler.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDeliveryScheduler.java
new file mode 100644
index 000000000..1d9f2a32c
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDeliveryScheduler.java
@@ -0,0 +1,376 @@
+package com.launchdarkly.sdk.android;
+
+import com.launchdarkly.logging.LDLogger;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Puts the event processor's deliveries on its scheduler thread, one at a time, and decides when that thread
+ * stops taking them.
+ *
+ * Every delivery runs there: a requested one, the periodic one, the retry after a failure, and the final one
+ * {@code close()} queues. So no two overlap, and each starts only once the one before it ends. This holds what
+ * that needs across threads: the requested delivery that is queued and has not started, the one retry, and the
+ * handover from running to shut down.
+ */
+final class EventDeliveryScheduler {
+ /** A requested delivery, given the future its callers wait on and who joined it by the time it began. */
+ interface RequestedDelivery {
+ void run(LDAwaitFuture
+ * A delivery that has not started yet will send everything recorded before it starts. That
+ * includes the caller's events, so waiting on it is as good as queuing a new one. Without this,
+ * flushes arriving faster than a post completes each queue their own, and the one that matters --
+ * the {@code flushAndWait} at shutdown -- waits behind all of them.
+ *
+ * @param answersACaller true if the caller will hear the outcome, and false if the caller discards it
+ * @param bypassBackoff true if the delivery, or the queued one it joins, is to post whatever wait the
+ * backoff is in
+ * @return what the delivery answers, or false at once if it is shutting down and nothing will be sent
+ */
+ Future
+ * Queueing and shutting down are one step under {@link #submitLock}, so that a flush cannot land in
+ * between them and put a delivery behind the release, where it would find the store closed.
+ *
+ * @param awaitFirst work still using those resources, such as the final commit, which the release waits
+ * for on the scheduler thread, never the caller's, however long a stalled disk makes it; may be null
+ * @param release run on the scheduler thread, or at once if that thread is already gone
+ */
+ void shutDown(Future awaitFirst, Runnable release) {
+ LDAwaitFuture
+ * Everything about the filesystem lives here: where the directory is, what the files in it are called, and
+ * how they are listed, read, moved and deleted. A store the application told to keep events off the disk
+ * has no {@code EventDirectory}, rather than one it is careful never to use.
+ *
+ * The directory's location is worked out on first use rather than at construction. A store that never
+ * reaches for the disk then makes no filesystem call at all, not even to find out where it would have
+ * written. That matters because the store is built on the thread that called {@code LDClient.init}, usually the main
+ * one: the platform creates the no-backup directory the first time it is asked for it.
+ *
+ * Belongs to an {@link EventStore}, which makes every call holding its {@code ioLock}, exactly as
+ * {@link DiskEventLog} belongs to this. The lock is handed down so that each call can check it is held,
+ * rather than each level taking a lock of its own that would have to be ordered against the others.
+ */
+class EventDirectory {
+ /**
+ * Names a log closed off for delivery, whichever version named it. What follows is the version of the
+ * rest of the name, {@link #BATCH_NAME_VERSION}, then the batch's payload ID and how many events are in
+ * it: {@code ready-v1-
+ * The count is in the name so that listing a batch this process did not close costs a directory entry
+ * rather than a read of the whole file. That is what every batch is at startup: the ones a previous run
+ * left behind, and the ones belonging to another process sharing the environment. Reading them all is
+ * done holding {@code ioLock}, which a commit at a commit point then waits behind.
+ *
+ * A batch is never appended to once it is closed, so the count is as good as the file is.
+ */
+ private static final String BATCH_PREFIX = "ready-";
+ /**
+ * The version of how the rest of a batch's name reads, so that a later version may change it without this
+ * one misreading the count. A batch another version named is never listed, counted or sent: reading it is
+ * that version's to do. Expiry still removes it, as it does any batch, so that a file this version does not
+ * read cannot stay on the device for good. A payload ID is a UUID, which cannot start with a {@code v}.
+ */
+ private static final String BATCH_NAME_VERSION = "v1-";
+ /** Names a batch this version named, and so reads. */
+ private static final String OWN_BATCH_PREFIX = BATCH_PREFIX + BATCH_NAME_VERSION;
+ private static final char NAME_FIELD_SEPARATOR = '.';
+ /**
+ * Names a log whose events are not yet in a batch: one its owner is appending to, or one left by an owner
+ * that is gone, for recovery to close off. The rest of the name is the UUID it was opened under.
+ */
+ static final String LOG_PREFIX = "pending-";
+ /**
+ * Names a log while it is being opened, before it is locked and has its header. Recovery never looks at
+ * these; one left by a process that died while opening it holds no events, and expiry removes it.
+ */
+ static final String OPENING_LOG_PREFIX = "new-";
+
+ /**
+ * The largest file this version reads. It writes none larger than {@link EventLogFormat#MAX_BATCH_SIZE},
+ * so one past this is not its own, whatever it holds. Read anyway, it would be held in memory whole, and
+ * one large enough would fail every pass that reached it, without ever being deleted. Twice the batch size,
+ * so that a later version may raise its own a little without an earlier one losing what it wrote.
+ */
+ static final long MAX_READ_SIZE = 2L * EventLogFormat.MAX_BATCH_SIZE;
+
+ /** A stored file larger than {@link #MAX_READ_SIZE}, which is never read and so can never be delivered. */
+ static final class TooLargeException extends IOException {
+ TooLargeException(long size) {
+ super("a stored file of " + size + " bytes, larger than this version reads");
+ }
+ }
+
+ /** A read that found too little memory for the file, which says nothing about the file itself. */
+ static final class MemoryShortException extends IOException {
+ MemoryShortException(long size, OutOfMemoryError cause) {
+ super("not enough memory to read " + size + " bytes", cause);
+ }
+ }
+ private static final String DIRECTORY_NAME = "com.launchdarkly.events";
+
+ /** Where one store keeps its files. */
+ interface Location {
+ File directory();
+ }
+
+ /** How a stored file is read, so a test can make a read fail while the file is intact. */
+ interface Reader {
+ FileInputStream openForReading(File file) throws IOException;
+ }
+
+ /**
+ * How many events, and how many bytes, something stored holds: the two things capacity counts. With how
+ * many of the events are summaries, which capacity leaves out, where this store wrote them.
+ */
+ static final class Tally {
+ static final Tally NONE = new Tally(0, 0);
+
+ final int events;
+ /** How many of {@link #events} are known to be summaries; none for a batch this store did not close. */
+ final int summaries;
+ final long bytes;
+
+ Tally(int events, long bytes) {
+ this(events, 0, bytes);
+ }
+
+ Tally(int events, int summaries, long bytes) {
+ this.events = events;
+ this.summaries = summaries;
+ this.bytes = bytes;
+ }
+
+ Tally plus(Tally other) {
+ return new Tally(events + other.events, summaries + other.summaries, bytes + other.bytes);
+ }
+ }
+
+ /** What discarding expired files did. */
+ static final class Discarded {
+ /** How many events went with them. */
+ final int events;
+ /** How much of that this process was counting, which is what stops counting against capacity. */
+ final Tally counted;
+
+ Discarded(int events, Tally counted) {
+ this.events = events;
+ this.counted = counted;
+ }
+ }
+
+ private final Location location;
+ private final Object ioLock;
+ private final LDLogger logger;
+ private final Reader reader;
+ private final DiskEventLog.Opener opener;
+
+ // Guarded by ioLock, and null until locateHoldingIoLock has run.
+ private File directory;
+ private DiskEventLog diskLog;
+ /** Whether {@link #openInThisProcess} counts this store, from locating the directory until closing. */
+ private boolean countedAsOpen;
+
+ /**
+ * What this process knows about each batch in the directory, by payload ID, which is how everything
+ * outside this class names one.
+ *
+ * The file is remembered rather than rebuilt from the payload ID, because the count is part of the name
+ * and a batch named without one has no count to rebuild it with. The count is remembered so that such
+ * a batch is read at most once.
+ *
+ * Only to be used while holding {@code ioLock}.
+ */
+ private final Map
+ * Left out of every listing, because a batch that is delivered and merely still on the disk would otherwise
+ * be found again by the next delivery, and the one after that, and sent each time. Each listing tries to
+ * delete it once more, so a directory that stops being read-only clears itself.
+ *
+ * Only for this session: a later run, which may well be able to delete the file, treats it as it would any
+ * other batch a previous run left behind. LaunchDarkly discards a repeated payload ID, so the cost of that is
+ * a wasted request rather than events counted twice.
+ *
+ * Only to be used while holding {@code ioLock}.
+ */
+ private final Set
+ * Only to be used while holding {@code ioLock}.
+ */
+ private final Map
+ * Requires {@code ioLock}.
+ */
+ File directory() {
+ locateHoldingIoLock();
+ return directory;
+ }
+
+ /** Requires {@code ioLock}. */
+ private DiskEventLog diskLog() {
+ locateHoldingIoLock();
+ return diskLog;
+ }
+
+ /**
+ * A batch is identified by its payload ID rather than by a path, so that a batch listed from the
+ * directory and the same batch as it was closed are one thing.
+ *
+ * Requires {@code ioLock}.
+ *
+ * @param events what the caller holding the batch knows it to contain, which is what names it where
+ * this process has neither closed nor listed it - as when it delivers a batch another process closed
+ */
+ private File batchFile(String payloadId, int events) {
+ Stored known = stored.get(payloadId);
+ return known == null ? newBatchFile(payloadId, events) : known.file;
+ }
+
+ /** Requires {@code ioLock}. */
+ private File newBatchFile(String payloadId, int events) {
+ return new File(directory(), OWN_BATCH_PREFIX + payloadId + NAME_FIELD_SEPARATOR + events);
+ }
+
+ /** A name for a batch of events not yet counted, which the first listing to read it counts. */
+ private File newUncountedBatchFile() {
+ return new File(directory(), OWN_BATCH_PREFIX + UUID.randomUUID());
+ }
+
+ /** Whether a file is a batch some other version named, which this one leaves to expiry. */
+ private static boolean isAnotherVersionsBatch(String fileName) {
+ return fileName.startsWith(BATCH_PREFIX) && !fileName.startsWith(OWN_BATCH_PREFIX);
+ }
+
+ /**
+ * The count in the name of a batch this version named, {@code ready-v1-
+ * Fields are split at {@link #NAME_FIELD_SEPARATOR}, which a payload ID never contains, so nothing
+ * about the count's own characters is needed to tell it apart. Fields after it are ignored.
+ *
+ * @return the count, or -1 where the name has none or it does not parse - a batch a recovery could not
+ * read, which is read once to count it
+ */
+ private static int eventsInName(String fileName) {
+ String[] fields = fileName.substring(OWN_BATCH_PREFIX.length()).split("\\" + NAME_FIELD_SEPARATOR, -1);
+ if (fields.length < 2 || fields[1].isEmpty()) {
+ return -1;
+ }
+ for (int i = 0; i < fields[1].length(); i++) {
+ char c = fields[1].charAt(i);
+ if (c < '0' || c > '9') {
+ return -1;
+ }
+ }
+ try {
+ return Integer.parseInt(fields[1]);
+ } catch (NumberFormatException e) {
+ return -1;
+ }
+ }
+
+ /** The payload ID in the name of a batch this version named: the first field, whatever follows it. */
+ private static String payloadIdInName(String fileName) {
+ String rest = fileName.substring(OWN_BATCH_PREFIX.length());
+ int end = rest.indexOf(NAME_FIELD_SEPARATOR);
+ return end < 0 ? rest : rest.substring(0, end);
+ }
+
+ /** A name for a new log in {@code directory}, under a UUID no other writer picks. */
+ static File newLogFile(File directory) {
+ return new File(directory, LOG_PREFIX + UUID.randomUUID());
+ }
+
+ /** Where {@code log} is created and given its header, before it is moved to its own name. */
+ static File openingFileFor(File log) {
+ return new File(log.getParentFile(), OPENING_LOG_PREFIX + log.getName().substring(LOG_PREFIX.length()));
+ }
+
+ // ---- writing ----
+
+ /**
+ * Appends to the open log, opening a new one where none is open.
+ *
+ * Requires {@code ioLock}.
+ *
+ * @return whether every byte landed. False means none of them is in the log, and that the caller should
+ * give up on the disk.
+ */
+ boolean append(byte[] bytes) {
+ assert Thread.holdsLock(ioLock);
+ try {
+ diskLog().append(bytes);
+ return true;
+ } catch (IOException | RuntimeException e) {
+ // Never let a full disk take the application down. The SDK gives up on persistence for the
+ // rest of the session instead, which is how other SDKs have crashed their hosts.
+ logger.warn("Giving up on persisting events: {}", LogValues.exceptionSummary(e));
+ return false;
+ }
+ }
+
+ /** What closing the open log produced. */
+ static final class LogClose {
+ /** The batch the log became, or null where the close failed. */
+ final EventStore.Batch batch;
+ /**
+ * Where the close failed, whether the log is still there to try again with; it is then still the
+ * open log, and its events still count as the open log's. Where it is not - a
+ * directory the platform reclaimed, say - its events are gone, and they have to stop counting
+ * against capacity, or the store would refuse events for the rest of the session.
+ */
+ final boolean logSurvives;
+ /**
+ * Where the log is gone, whether nothing could have delivered its events: it was still held, so no
+ * recovery could take it. One no longer held, as persistence given up on leaves it, may have been
+ * moved into a batch by a recovery, which delivers it.
+ */
+ final boolean deletedUnsent;
+
+ private LogClose(EventStore.Batch batch, boolean logSurvives, boolean deletedUnsent) {
+ this.batch = batch;
+ this.logSurvives = logSurvives;
+ this.deletedUnsent = deletedUnsent;
+ }
+ }
+
+ /**
+ * Closes the open log into a batch, so the next append starts a new one.
+ *
+ * Requires {@code ioLock}.
+ *
+ * @param events how many events the log holds, which the caller has counted as it staged them
+ * @param summaries how many of {@code events} are summaries
+ * @param bytes how big the log is, which the caller has counted as it wrote it
+ */
+ LogClose closeLog(int events, int summaries, long bytes) {
+ assert Thread.holdsLock(ioLock);
+ String payloadId = UUID.randomUUID().toString();
+ File file = newBatchFile(payloadId, events);
+ // Asked before the move, which closes a log it finds gone.
+ boolean held = diskLog().isHeld();
+ if (!diskLog().closeMovingTo(file)) {
+ logger.warn("Could not close the event log for delivery");
+ boolean survives = diskLog().exists();
+ return new LogClose(null, survives, !survives && held);
+ }
+
+ stored.put(payloadId, new Stored(file, events, summaries, bytes, System.currentTimeMillis()));
+ return new LogClose(new EventStore.Batch(payloadId, events), true, false);
+ }
+
+ /**
+ * Closes the descriptor, leaving everything where it is, and stops counting the directory as open in this
+ * process. Requires {@code ioLock}.
+ */
+ void closeOpenLog() {
+ assert Thread.holdsLock(ioLock);
+ // Not diskLog(): a store that never wrote has never asked the platform where it lives, and shutting
+ // down is no reason to start.
+ if (diskLog != null) {
+ diskLog.closeOutput();
+ }
+ if (countedAsOpen) {
+ countedAsOpen = false;
+ synchronized (openInThisProcess) {
+ String path = directory.getAbsolutePath();
+ Integer stores = openInThisProcess.get(path);
+ if (stores != null && stores > 1) {
+ openInThisProcess.put(path, stores - 1);
+ } else {
+ openInThisProcess.remove(path);
+ }
+ }
+ }
+ }
+
+ /**
+ * Locks a batch against expiry, in this process and every other, for as long as a request for it is out.
+ * Each call is ended by one {@link #stoppedSending}. Requires {@code ioLock}.
+ *
+ * @param events what the caller holding the batch knows it to contain, as for {@link #payload}
+ */
+ void startedSending(String payloadId, int events) {
+ assert Thread.holdsLock(ioLock);
+ Sending batch = sending.get(payloadId);
+ if (batch == null) {
+ batch = new Sending(batchFile(payloadId, events));
+ sending.put(payloadId, batch);
+ }
+ batch.senders++;
+ BatchLocks.startedSending(batch.file);
+ }
+
+ /** Ends one {@link #startedSending}. Requires {@code ioLock}. */
+ void stoppedSending(String payloadId) {
+ assert Thread.holdsLock(ioLock);
+ Sending batch = sending.get(payloadId);
+ if (batch == null) {
+ return;
+ }
+ BatchLocks.stoppedSending(batch.file);
+ if (--batch.senders == 0) {
+ sending.remove(payloadId);
+ }
+ }
+
+ // ---- listing and reading ----
+
+ /**
+ * Requires {@code ioLock}.
+ *
+ * Sizes come from the directory entries, so that counting the directory's bytes reads no file either.
+ *
+ * A directory that is not there holds nothing. One that is there and cannot be listed - out of file
+ * descriptors, or no longer readable - is not empty, so it answers with the batches the last listing or
+ * close counted, as they were counted, and says the listing is incomplete.
+ *
+ * @return the batches the directory holds, oldest first, which is the order they are delivered in
+ */
+ EventStore.Listing listBatches() {
+ assert Thread.holdsLock(ioLock);
+ List
+ * Requires {@code ioLock}.
+ */
+ private boolean isDeliveredButUndeleted(String payloadId, File file) {
+ if (!deliveredButUndeleted.contains(payloadId)) {
+ return false;
+ }
+ tryDelete(file);
+ if (!file.exists()) {
+ deliveredButUndeleted.remove(payloadId);
+ }
+ return true;
+ }
+
+ /**
+ * Requires {@code ioLock}.
+ *
+ * @return the JSON payload for a batch, assembled in the array the file was read into; or null if it is
+ * gone or holds no events
+ * @throws EventStore.UnreadableBatchException if the batch is there but holds nothing this version can
+ * send, which includes one larger than {@link #MAX_READ_SIZE}
+ * @throws IOException if the batch is there but could not be read this time
+ */
+ EventLogFormat.Payload payload(String payloadId, int events) throws IOException {
+ assert Thread.holdsLock(ioLock);
+ byte[] log;
+ try {
+ log = readFile(batchFile(payloadId, events));
+ } catch (TooLargeException e) {
+ throw new EventStore.UnreadableBatchException(payloadId);
+ }
+ if (log == null) {
+ return null;
+ }
+ // No more than the batch was closed holding. Writing that failed and could not be truncated
+ // leaves whole frames after them, whose events are delivered from memory instead, and would
+ // otherwise be sent twice.
+ EventLogFormat.Payload payload =
+ EventLogFormat.assemblePayloadInPlace(log, events > 0 ? events : Integer.MAX_VALUE);
+ // A log this version reads that holds no events has nothing to lose, unless its first frame is
+ // damaged. Where there is no payload the log is as it was read.
+ if (payload == null && (EventLogFormat.eventCount(log) != 0 || EventLogFormat.isDamaged(log))) {
+ throw new EventStore.UnreadableBatchException(payloadId);
+ }
+ return payload;
+ }
+
+ /**
+ * Removes a batch. Requires {@code ioLock}.
+ *
+ * @return what it stopped counting, which is nothing for a batch that was already gone - one another
+ * process delivered, say, which a listing has stopped counting already
+ */
+ Tally remove(String payloadId, int events) {
+ assert Thread.holdsLock(ioLock);
+ File file = batchFile(payloadId, events);
+ tryDelete(file);
+ if (file.exists()) {
+ // Its events stop counting all the same: LaunchDarkly has them, and holding capacity for a file this
+ // process may never be allowed to delete would cost the session events it could still have recorded.
+ logger.debug("Could not delete a delivered batch; it will not be sent again this session");
+ deliveredButUndeleted.add(payloadId);
+ }
+ Stored removed = stored.remove(payloadId);
+ return removed == null ? Tally.NONE : removed.tally();
+ }
+
+ /**
+ * Every event in the directory, encoded exactly as it will be sent. Requires {@code ioLock}.
+ *
+ * For tests and for diagnosing a store rather than for the recording path: it reads every file.
+ */
+ List
+ * Each file's modification time is read once, before the sort, so that the order cannot change under it.
+ *
+ * @return every batch file in the directory, oldest first; empty if the directory is not there, and null if
+ * it is there and could not be listed
+ */
+ private List
+ * Whoever recorded the events in such a log is gone, so there is no one left to add to it. That may be a
+ * previous run of this process, another process of the application, or a store in this one that has since
+ * closed. A log some live process holds, this store's own included, is left alone. None of those listed
+ * becomes held later: a log is opened under a name of its own and never reopened once closed.
+ *
+ * Requires {@code ioLock}.
+ */
+ List
+ * A batch's modification time is when its newest event was written, since a batch is renamed rather than
+ * rewritten when it is closed, so a batch is discarded only once everything in it is that old. A log is
+ * usually recovered rather than left to grow old, so one that gets here is one recovery could not move.
+ * A batch another version named is deleted too, without being read or counted.
+ *
+ * A batch whose request is out is left alone however old it is, whichever store or process sent it; see
+ * {@link BatchLocks}.
+ *
+ * @param sending batches whose request is out from this store
+ */
+ Discarded discard(long cutoffMillis, Set
+ * Age decides rather than the keys the application configures, because another process of the same
+ * application may be using a key this one does not configure. Nothing is touched here that this store
+ * reads or writes, so it needs no lock. A directory is removed only if it is empty, so a log another
+ * process starts in it meanwhile keeps the directory and itself.
+ *
+ * Only where {@code current} is where the SDK puts environments, so that a store handed some other
+ * directory never goes through that directory's siblings. The directory of a store open in this process
+ * is left alone, and so is a batch some live process is sending; see {@link BatchLocks}.
+ *
+ * @return how many events were discarded
+ */
+ static int discardAbandonedEnvironments(File current, long cutoffMillis) {
+ File root = current.getParentFile();
+ if (root == null || !DIRECTORY_NAME.equals(root.getName())) {
+ return 0;
+ }
+ File[] environments = root.listFiles();
+ if (environments == null) {
+ return 0;
+ }
+ int events = 0;
+ for (File environment : environments) {
+ if (environment.equals(current) || !environment.isDirectory() || isOpenInThisProcess(environment)) {
+ continue;
+ }
+ File[] files = environment.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ // As in discard(): a file that fails is left for a later pass.
+ try {
+ if (isWrittenSince(file, cutoffMillis)) {
+ continue;
+ }
+ String name = file.getName();
+ if (isAnotherVersionsBatch(name)) {
+ BatchLocks.deleteUnlessBeingSent(file);
+ } else if (name.startsWith(OWN_BATCH_PREFIX)) {
+ if (BatchLocks.isSentInThisProcess(file)) {
+ continue;
+ }
+ int named = eventsInName(name);
+ int held = named >= 0 ? named : eventsReading(file);
+ if (held >= 0 && BatchLocks.deleteUnlessBeingSent(file)) {
+ events += held;
+ }
+ } else {
+ events += discardIfAbandonedLog(file);
+ }
+ } catch (RuntimeException ignored) {
+ }
+ }
+ }
+ // Removes the directory only if it is empty.
+ delete(environment);
+ }
+ return events;
+ }
+
+ /**
+ * A file whose modification time is unknown counts as written since, so it is never taken for an old one.
+ */
+ private static boolean isWrittenSince(File file, long cutoffMillis) {
+ long modified = file.lastModified();
+ return modified <= 0 || modified >= cutoffMillis;
+ }
+
+ /**
+ * Deletes a log no live process holds, or one a process died opening, which holds no events.
+ *
+ * @return how many events went with it
+ */
+ private static int discardIfAbandonedLog(File file) {
+ String name = file.getName();
+ if (name.startsWith(OPENING_LOG_PREFIX)) {
+ delete(file);
+ return 0;
+ }
+ if (!name.startsWith(LOG_PREFIX) || DiskEventLog.isHeldByLiveProcess(file)) {
+ return 0;
+ }
+ int held = eventsReading(file);
+ return held >= 0 && delete(file) ? held : 0;
+ }
+
+ /**
+ * @return how many events a file being discarded holds, or -1 if it could not be read, in which case it is
+ * left for a later pass: deleted now, its events would go uncounted. One larger than
+ * {@link #MAX_READ_SIZE} counts as one, the fewest it can hold: it will never be read, and left, it would
+ * be tried by every pass.
+ */
+ // Files.newInputStream needs API 26.
+ @SuppressWarnings("IOStreamConstructor")
+ private static int eventsReading(File file) {
+ byte[] contents;
+ try (FileInputStream in = new FileInputStream(file)) {
+ contents = readAll(in);
+ } catch (TooLargeException e) {
+ return 1;
+ } catch (IOException e) {
+ return -1;
+ }
+ return Math.max(0, EventLogFormat.eventCount(contents));
+ }
+
+ private static boolean delete(File file) {
+ return file.delete() || !file.exists();
+ }
+
+ // ---- naming ----
+
+ /** The directory the store for one environment belongs in. */
+ static File defaultDirectory(File noBackupFilesDir, String mobileKey) {
+ return new File(new File(noBackupFilesDir, DIRECTORY_NAME), environmentDirectoryName(mobileKey));
+ }
+
+ /**
+ * Names the directory for one environment, hashed so the mobile key itself is never a filename.
+ * Not {@link LDUtil#urlSafeBase64Hash(String)}, for the reasons {@link LDUtil#hexDigest} gives.
+ */
+ private static String environmentDirectoryName(String mobileKey) {
+ // Half a digest, which is far more than enough to keep two mobile keys apart.
+ return LDUtil.hexDigest(mobileKey, 16);
+ }
+}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventLogFormat.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventLogFormat.java
new file mode 100644
index 000000000..3a902162c
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventLogFormat.java
@@ -0,0 +1,365 @@
+package com.launchdarkly.sdk.android;
+
+import androidx.annotation.VisibleForTesting;
+
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.zip.CRC32;
+
+/**
+ * The on-disk shape of an event log.
+ *
+ * A log opens with a magic and a format version, and holds a sequence of frames:
+ *
+ * The format is byte-for-byte the one the iOS SDK writes, so the two can be reasoned about, and
+ * debugged, as one thing.
+ */
+final class EventLogFormat {
+ /** Bump this whenever the framing or the meaning of a frame changes. */
+ static final int VERSION = 1;
+ static final byte[] MAGIC = {'L', 'D', 'E', 'V'};
+ static final int FILE_HEADER_SIZE = 6;
+ /** The type and the size, before a frame's payload. */
+ static final int FRAME_HEADER_SIZE = 6;
+ /** The checksum, after a frame's payload. */
+ static final int FRAME_TRAILER_SIZE = 4;
+ /** What a frame takes beyond its payload. */
+ static final int FRAME_OVERHEAD = FRAME_HEADER_SIZE + FRAME_TRAILER_SIZE;
+ /** The only frame type written today; a reader skips a frame whose type it does not know. */
+ static final int EVENT_FRAME = 1;
+ /**
+ * The most a batch may hold, file header included, so that no request it becomes is larger than the
+ * events service accepts.
+ */
+ static final int MAX_BATCH_SIZE = 2 * 1024 * 1024;
+ /**
+ * A ceiling on a single frame's payload: the most that still fits in a batch on its own. It is also
+ * what keeps a corrupt size from making recovery allocate wildly.
+ */
+ static final int MAX_FRAME_SIZE = MAX_BATCH_SIZE - FILE_HEADER_SIZE - FRAME_OVERHEAD;
+
+ private EventLogFormat() {
+ }
+
+ static byte[] fileHeader() {
+ byte[] header = new byte[FILE_HEADER_SIZE];
+ System.arraycopy(MAGIC, 0, header, 0, MAGIC.length);
+ header[4] = (byte) (VERSION >> 8);
+ header[5] = (byte) VERSION;
+ return header;
+ }
+
+ static void writeFrame(ByteArrayOutputStream target, byte[] serializedEvent) {
+ int length = serializedEvent.length;
+ byte[] header = new byte[FRAME_HEADER_SIZE];
+ header[0] = (byte) (EVENT_FRAME >> 8);
+ header[1] = (byte) EVENT_FRAME;
+ writeInt32(header, 2, length);
+ CRC32 checksum = new CRC32();
+ checksum.update(header, 0, FRAME_HEADER_SIZE);
+ checksum.update(serializedEvent, 0, length);
+ byte[] trailer = new byte[FRAME_TRAILER_SIZE];
+ writeInt32(trailer, 0, (int) checksum.getValue());
+ target.write(header, 0, FRAME_HEADER_SIZE);
+ target.write(serializedEvent, 0, length);
+ target.write(trailer, 0, FRAME_TRAILER_SIZE);
+ }
+
+ /**
+ * @return whether the log holds frames that cannot be read back: torn, or failing their
+ * checksum; false for a log this version does not read at all
+ */
+ static boolean isDamaged(byte[] log) {
+ Frames frames = new Frames(log);
+ if (!frames.isReadable()) {
+ return false;
+ }
+ while (frames.hasNext()) {
+ frames.next();
+ }
+ return !frames.reachedEnd();
+ }
+
+ /** Frames on their own, as a log: only for the in-memory fallback, which has no file. */
+ static byte[] withFileHeader(byte[] frames) {
+ return withFileHeader(frames, 0, frames.length);
+ }
+
+ /** The frames from {@code start} to {@code end}, as a log of their own. */
+ static byte[] withFileHeader(byte[] frames, int start, int end) {
+ byte[] log = new byte[FILE_HEADER_SIZE + end - start];
+ System.arraycopy(fileHeader(), 0, log, 0, FILE_HEADER_SIZE);
+ System.arraycopy(frames, start, log, FILE_HEADER_SIZE, end - start);
+ return log;
+ }
+
+ /**
+ * @return how many event frames the log holds, or -1 if it is not a log this version reads
+ */
+ static int eventCount(byte[] log) {
+ Frames frames = new Frames(log);
+ if (!frames.isReadable()) {
+ return -1;
+ }
+ int events = 0;
+ while (frames.hasNext()) {
+ if (frames.next().type == EVENT_FRAME) {
+ events++;
+ }
+ }
+ return events;
+ }
+
+ /**
+ * A JSON array of events at the start of an array that may run on past it, so that it can be sent
+ * from where it was assembled rather than copied out first.
+ */
+ static final class Payload {
+ final byte[] bytes;
+ final int length;
+ /**
+ * How many events made it in: fewer than the batch holds where a frame after the first is torn or
+ * fails its checksum, since assembly stops there.
+ */
+ final int eventCount;
+
+ Payload(byte[] bytes, int length, int eventCount) {
+ this.bytes = bytes;
+ this.length = length;
+ this.eventCount = eventCount;
+ }
+
+ byte[] toByteArray() {
+ return Arrays.copyOf(bytes, length);
+ }
+ }
+
+ /**
+ * Turns the log into the JSON array LaunchDarkly expects, in the log's own array, without parsing
+ * the events: they were serialized on the way in and are shipped exactly as they were recorded.
+ *
+ * The array always has room: the file header becomes the '[', and each frame's header and trailer
+ * the ',' or ']' after its payload. Each payload moves towards the start only once its frame has been checked, and
+ * never past where that frame began, so no frame still to be checked is written over.
+ *
+ * @return the payload of the first {@code maxEvents} events, leaving out any after them; or null if the
+ * log is unreadable or holds no events, in which case the log is left as it was
+ */
+ static Payload assemblePayloadInPlace(byte[] log, int maxEvents) {
+ Frames frames = new Frames(log);
+ if (!frames.isReadable()) {
+ return null;
+ }
+ int events = 0;
+ int cursor = 1;
+ while (events < maxEvents && frames.hasNext()) {
+ Frame frame = frames.next();
+ if (frame.type != EVENT_FRAME) {
+ continue;
+ }
+ System.arraycopy(log, frame.start, log, cursor, frame.length);
+ cursor += frame.length;
+ log[cursor++] = ',';
+ events++;
+ }
+ if (events == 0) {
+ return null;
+ }
+ log[0] = '[';
+ log[cursor - 1] = ']';
+ return new Payload(log, cursor, events);
+ }
+
+ /**
+ * {@link #assemblePayloadInPlace} into an array of its own, leaving the log as it was.
+ *
+ * @return the payload, or null if the log is unreadable or holds no events
+ */
+ @VisibleForTesting
+ static byte[] assembleBody(byte[] log) {
+ return assembleBody(log, Integer.MAX_VALUE);
+ }
+
+ @VisibleForTesting
+ static byte[] assembleBody(byte[] log, int maxEvents) {
+ Payload payload = assemblePayloadInPlace(log.clone(), maxEvents);
+ return payload == null ? null : payload.toByteArray();
+ }
+
+ /**
+ * @return each event in the log, serialized as it will be sent; empty if the log is unreadable
+ */
+ static List
+ * A process that died partway through a write leaves a torn frame at the end. Everything before it
+ * is intact, so recovery keeps that and drops only the tail. A frame the storage damaged afterwards
+ * ends the walk the same way, because its size can no longer be trusted to say where the next frame
+ * starts.
+ *
+ * There is no next frame from the first one that is not wholly there or fails its checksum, and none
+ * at all in a log that is not {@link #isReadable() readable}. {@link #hasNext()} is what checks the
+ * next frame, so a caller may write anywhere before where the last frame it was handed ends.
+ */
+ static final class Frames implements Iterator {
+ private final byte[] log;
+ private final boolean readable;
+ private final CRC32 checksum = new CRC32();
+ private int cursor = FILE_HEADER_SIZE;
+ private boolean stopped;
+ private Frame checked;
+
+ Frames(byte[] log) {
+ this.log = log;
+ this.readable = isReadable(log);
+ }
+
+ /** Whether this is a log this version reads at all. */
+ boolean isReadable() {
+ return readable;
+ }
+
+ /**
+ * Whether the walk ended at the end of the log, rather than at a frame that is torn or fails its
+ * checksum. Only meaningful once {@link #hasNext()} has returned false.
+ */
+ boolean reachedEnd() {
+ return !stopped && cursor == log.length;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (checked != null) {
+ return true;
+ }
+ if (!readable || stopped || cursor + FRAME_OVERHEAD > log.length) {
+ return false;
+ }
+ int type = readUInt16(log, cursor);
+ int length = readInt32(log, cursor + 2);
+ int start = cursor + FRAME_HEADER_SIZE;
+ if (length <= 0 || length > MAX_FRAME_SIZE || start + length + FRAME_TRAILER_SIZE > log.length) {
+ stopped = true;
+ return false;
+ }
+ int end = start + length;
+ checksum.reset();
+ checksum.update(log, cursor, end - cursor);
+ if ((int) checksum.getValue() != readInt32(log, end)) {
+ stopped = true;
+ return false;
+ }
+ checked = new Frame(type, start, length);
+ cursor = end + FRAME_TRAILER_SIZE;
+ return true;
+ }
+
+ @Override
+ public Frame next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ Frame frame = checked;
+ checked = null;
+ return frame;
+ }
+
+ private static boolean isReadable(byte[] log) {
+ if (log == null || log.length < FILE_HEADER_SIZE) {
+ return false;
+ }
+ for (int i = 0; i < MAGIC.length; i++) {
+ if (log[i] != MAGIC[i]) {
+ return false;
+ }
+ }
+ return readUInt16(log, MAGIC.length) == VERSION;
+ }
+ }
+
+ /** The size, header and trailer included, of the frame at {@code start} in frames this process wrote. */
+ static int frameSize(byte[] frames, int start) {
+ return FRAME_OVERHEAD + readInt32(frames, start + 2);
+ }
+
+ /**
+ * Where a run of whole frames from {@code start}, in frames this process wrote, ends if it is to take
+ * no more than {@code room} bytes. It takes the first frame whatever its size.
+ */
+ static int endOfFramesFitting(byte[] frames, int start, long room) {
+ int end = start + frameSize(frames, start);
+ while (end < frames.length && end + frameSize(frames, end) - start <= room) {
+ end += frameSize(frames, end);
+ }
+ return end;
+ }
+
+ /** How many frames there are between two frame boundaries in frames this process wrote. */
+ static int frameCount(byte[] frames, int start, int end) {
+ int count = 0;
+ for (int cursor = start; cursor < end; cursor += frameSize(frames, cursor)) {
+ count++;
+ }
+ return count;
+ }
+
+ private static int readUInt16(byte[] bytes, int offset) {
+ return ((bytes[offset] & 0xff) << 8) | (bytes[offset + 1] & 0xff);
+ }
+
+ static int readInt32(byte[] bytes, int offset) {
+ return ((bytes[offset] & 0xff) << 24) | ((bytes[offset + 1] & 0xff) << 16)
+ | ((bytes[offset + 2] & 0xff) << 8) | (bytes[offset + 3] & 0xff);
+ }
+
+ private static void writeInt32(byte[] bytes, int offset, int value) {
+ bytes[offset] = (byte) (value >> 24);
+ bytes[offset + 1] = (byte) (value >> 16);
+ bytes[offset + 2] = (byte) (value >> 8);
+ bytes[offset + 3] = (byte) value;
+ }
+}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventStore.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventStore.java
new file mode 100644
index 000000000..e3c6555cf
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventStore.java
@@ -0,0 +1,1333 @@
+package com.launchdarkly.sdk.android;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.VisibleForTesting;
+
+import com.launchdarkly.logging.LDLogger;
+import com.launchdarkly.logging.LogValues;
+import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * A crash-durable append log of serialized analytics events.
+ *
+ * Events are appended to an open log, which is closed into a batch when the event processor is ready to
+ * deliver one. A batch is deleted only once LaunchDarkly has accepted it, so a delivery interrupted by
+ * the process dying is retried on the next run rather than lost.
+ *
+ * Bytes reach the kernel through an ordinary write and are never {@code fsync}ed. What this defends
+ * against is the process dying, which is what an uncaught exception, an ANR kill, a native crash, and
+ * the system reclaiming a backgrounded application all are. A process cannot take back bytes the
+ * kernel already holds. Only losing the kernel itself, to a panic or a power cut, can discard them, and
+ * covering that would mean an {@code fsync} per event, which costs milliseconds where this costs
+ * microseconds.
+ *
+ * Several clients. An application configured for more than one environment gets one
+ * {@code LDClient} and one store per mobile key, and the mobile key's hash is part of the directory, so
+ * two environments never read or write each other's events.
+ *
+ * Several processes. An application may declare components in more than one process, and each
+ * gets its own copy of the SDK that shares nothing but the filesystem. Only the open log is per-process,
+ * named after the process appending to it. That keeps two processes from interleaving frames in one file,
+ * and keeps one process's startup recovery from stealing a log another is still writing to.
+ * Closed batches are deliberately left shared: a process that recorded events and then died may never
+ * run again, and a batch nobody but that process could see would never be delivered. If two processes
+ * do deliver the same batch, its payload ID is the same both times, and LaunchDarkly discards a repeated
+ * payload ID rather than counting the events twice.
+ *
+ * What a log looks like on disk is {@link EventLogFormat}; the log this process appends to is
+ * {@link DiskEventLog}; the directory the two of them live in is {@link EventDirectory}. This class is
+ * what decides when events move between memory, that directory, and the batches closed off for delivery,
+ * and what counts them against capacity on the way. It has no directory at all where the application asked
+ * for events to be kept off the disk, which is what keeps the filesystem out of that case entirely.
+ */
+class EventStore implements Closeable {
+ /**
+ * Where the events are kept, or null where the application asked for them to be kept off the disk.
+ *
+ * Everything about the filesystem is behind this, so that the store itself is the buffer, the capacity
+ * arithmetic, and the batches it holds in memory. A store with no disk runs the way the SDK ran before
+ * persistence existed, and never touches the filesystem, not even to look for a previous run's events.
+ */
+ private final EventDirectory disk;
+ private final int capacity;
+ /** The most bytes summaries may take the store to. */
+ private final long byteCapacity;
+ /**
+ * The most bytes full events may take the store to: the byte capacity less a fifth of it, held back so
+ * that summaries, which bypass {@link #capacity}, still have room once full events are being refused.
+ */
+ private final long eventByteCapacity;
+ /** How long a batch may go unwritten before it is discarded rather than delivered. */
+ private final long maxAgeMillis;
+
+ /**
+ * Guards the buffer and the counters. Held for a copy and never while writing, so an
+ * evaluation recording an event waits on another thread's array copy at worst, never on the disk.
+ */
+ private final Object bufferLock = new Object();
+ /**
+ * Serializes writes so that two threads committing at once cannot interleave a frame. Taken before
+ * {@link #bufferLock}, never after it.
+ */
+ private final Object ioLock = new Object();
+
+ // Guarded by bufferLock.
+ /** Full events staged, {@link #bufferedEventCount} of them. */
+ private final FrameBuffer bufferData = new FrameBuffer();
+ private int bufferedEventCount;
+ /**
+ * Summaries staged, {@link #bufferedSummaryCount} of them, apart from the full events because capacity
+ * does not count them. Everything staged is taken as the full events followed by the summaries, so that
+ * any part of it can say how many of its frames are summaries from where the summaries start.
+ */
+ private final FrameBuffer summaryData = new FrameBuffer();
+ private int bufferedSummaryCount;
+ /**
+ * How many of the frames counted in each state below are summaries, state for state, so capacity can
+ * leave them out. Changed in the same critical sections as the counts beside them. A batch this store
+ * did not close - one a previous run or another process left - counts none: its name says how many
+ * frames it holds, not what they are, so every one counts as an event, which errs towards refusing.
+ */
+ private int inTransitSummaries;
+ private int committedSummaries;
+ private int closedSummaries;
+ private int sendingSummaries;
+ /**
+ * Events taken out of the buffer to be written to the log, or assembled into a batch held in
+ * memory, and not yet counted where they land. Both happen outside this lock, and for as long as
+ * they take the events are in neither place; without this they would be counted nowhere, and
+ * capacity would admit that many more.
+ */
+ private int inTransitEvents;
+ private int committedEvents;
+ /**
+ * Events in closed batches: those the {@link #disk} is holding plus those in {@link #inMemoryBatches}.
+ * Changed only in the critical section that changes one of those, so that it cannot drift from them; a
+ * total worked out from them and written afterward would overwrite a close or a removal in between.
+ */
+ private int closedEvents;
+ /**
+ * Closed batches whose request is out, by payload ID, and the events in them as {@link #closedEvents}
+ * counts them. Every one is also counted there: one is only added while it is counted, and leaves in the
+ * same critical section that stops counting it, so {@link #sendingEvents} can never exceed
+ * {@link #closedEvents}.
+ */
+ private final Map
+ * Written under {@link #bufferLock} but volatile, so {@link #isPersisting} can answer without it:
+ * every {@code track} asks, and a commit takes that lock once for each event it stages.
+ */
+ private volatile boolean persistEvents;
+
+ /**
+ * Guards {@link #inMemoryBatches}.
+ *
+ * Its own lock rather than {@code ioLock}, so that a store the application told to keep off the disk
+ * never waits on the lock disk I/O is serialized under; there, held batches are the only ones there
+ * are. Taken inside {@code ioLock} where there is a disk, never the other way round, and taken
+ * before {@code bufferLock}, never after it.
+ */
+ private final Object heldLock = new Object();
+ /**
+ * Batches held in memory because the filesystem would not take them, or because it was never asked.
+ *
+ * Persistence failing should cost durability and nothing else, so the store falls back to what the
+ * SDK did before it existed: hold the events, deliver them, lose them only if the process dies.
+ *
+ * Only to be used while holding {@link #heldLock}.
+ */
+ private final Map
+ * Events reserved while being encoded have no bytes yet; they are counted when they are staged.
+ */
+ private long countedBytesHoldingBufferLock() {
+ return bufferData.size() + summaryData.size() + inTransitBytes + committedBytes + (closedBytes - sendingBytes);
+ }
+
+ /**
+ * Whether full events can still be admitted by size, asked before an event is kept for encoding so
+ * that one there is no room for is refused without the cost of encoding it. Staging checks again
+ * against the event's actual size.
+ */
+ boolean hasRoomForEventBytes() {
+ synchronized (bufferLock) {
+ return countedBytesHoldingBufferLock() < eventByteCapacity;
+ }
+ }
+
+ /**
+ * {@link #getPendingEventCount} and {@link #hasRoomForEventBytes} in one acquisition of the lock, for the
+ * check every recorded event makes, where the lock is contended by the commit staging events.
+ *
+ * @param heldElsewhere events the caller holds that the store does not count yet
+ * @param capacity the most events to hold, counting {@code heldElsewhere}
+ * @return whether one more full event can be admitted, by count and by bytes
+ */
+ boolean admitsEvent(int heldElsewhere, int capacity) {
+ synchronized (bufferLock) {
+ return heldElsewhere + countedEventsHoldingBufferLock() < capacity
+ && countedBytesHoldingBufferLock() < eventByteCapacity;
+ }
+ }
+
+ /**
+ * Counts events against capacity before they are staged, for a caller that has taken them from
+ * somewhere capacity was already counting them and will encode them before staging.
+ *
+ * Each {@link #stageReserved} or {@link #writeReservedAlone} uses one up, and
+ * {@link #releaseReservations} gives back any that never made it, such as an event that could not be
+ * serialized.
+ */
+ void reserve(int events) {
+ synchronized (bufferLock) {
+ reservedEvents += events;
+ }
+ }
+
+ /**
+ * Gives back {@code events} reservations not used up by {@link #stageReserved} or
+ * {@link #writeReservedAlone}. Only the caller's own, since another caller may hold reservations
+ * alongside.
+ */
+ void releaseReservations(int events) {
+ synchronized (bufferLock) {
+ reservedEvents = Math.max(0, reservedEvents - events);
+ }
+ }
+
+ /**
+ * @return whether a commit writes anything, which is false where persistence is off and once writing
+ * has failed and the store has given up on it
+ */
+ boolean isPersisting() {
+ return persistEvents;
+ }
+
+ /**
+ * Stages an already serialized event.
+ *
+ * Staging is a copy into a buffer and never writes, however much accumulates; {@link #commit()} is what
+ * makes the event outlive the process, so how many bytes one write carries is the caller's decision.
+ *
+ * @return false if the event was dropped because capacity or the byte capacity is reached
+ */
+ boolean stage(byte[] serializedEvent) {
+ return stage(serializedEvent, false);
+ }
+
+ /**
+ * @param bypassingCapacity true for a summary, which must be recorded even when the store is full of
+ * events, and may use the bytes held back from them; it is still refused once the byte capacity
+ * itself is reached, whether or not events are persisted
+ */
+ boolean stage(byte[] serializedEvent, boolean bypassingCapacity) {
+ return stage(serializedEvent, bypassingCapacity, false);
+ }
+
+ /**
+ * Stages an event {@link #reserve} already counted, turning its reservation into a staged event in
+ * one step. Bypasses capacity, which the reservation already answered to, but not the bytes left for
+ * full events, which only the event's encoded size can answer.
+ */
+ boolean stageReserved(byte[] serializedEvent) {
+ return stage(serializedEvent, true, true);
+ }
+
+ private boolean stage(byte[] serializedEvent, boolean bypassingCapacity, boolean reserved) {
+ if (serializedEvent == null || serializedEvent.length == 0
+ || serializedEvent.length > EventLogFormat.MAX_FRAME_SIZE) {
+ if (reserved) {
+ releaseReservations(1);
+ }
+ return false;
+ }
+ long frameSize = (long) serializedEvent.length + EventLogFormat.FRAME_OVERHEAD;
+ // Only a summary bypasses capacity without a reservation, and only a summary may use the bytes held
+ // back.
+ boolean isSummary = bypassingCapacity && !reserved;
+
+ synchronized (bufferLock) {
+ // In the same critical section as the append, so the event is never counted twice or not at all.
+ if (reserved && reservedEvents > 0) {
+ reservedEvents--;
+ }
+ if (!bypassingCapacity && countedEventsHoldingBufferLock() >= capacity) {
+ return false;
+ }
+ // Bytes bound events kept off the disk too: capacity counts events, so held in memory through an
+ // outage it would allow that many whatever size they are, and a summary, which capacity never
+ // refuses, would otherwise go on growing for the life of the process.
+ if (countedBytesHoldingBufferLock() + frameSize > (isSummary ? byteCapacity : eventByteCapacity)) {
+ return false;
+ }
+ if (isSummary) {
+ EventLogFormat.writeFrame(summaryData, serializedEvent);
+ bufferedSummaryCount++;
+ } else {
+ EventLogFormat.writeFrame(bufferData, serializedEvent);
+ bufferedEventCount++;
+ }
+ }
+ return true;
+ }
+
+ /** A buffer of frames that copies into a larger array without a copy of its own on the way. */
+ private static final class FrameBuffer extends ByteArrayOutputStream {
+ void copyInto(byte[] target, int offset) {
+ System.arraycopy(buf, 0, target, offset, count);
+ }
+ }
+
+ /**
+ * Everything staged, taken out of the buffers in one step: the full events, then the summaries.
+ */
+ private static final class Staged {
+ final byte[] frames;
+ /** Where in {@link #frames} the summaries start. */
+ final int summariesFrom;
+ /** Every frame in {@link #frames}, summaries included. */
+ final int events;
+ final int summaries;
+
+ Staged(byte[] frames, int summariesFrom, int events, int summaries) {
+ this.frames = frames;
+ this.summariesFrom = summariesFrom;
+ this.events = events;
+ this.summaries = summaries;
+ }
+
+ /** How many of the frames from {@code from} up to {@code to} are summaries. */
+ int summariesIn(int from, int to) {
+ return to <= summariesFrom ? 0 : EventLogFormat.frameCount(frames, Math.max(from, summariesFrom), to);
+ }
+ }
+
+ /**
+ * Takes everything staged, counting it in transit until it is counted where it lands. Requires
+ * {@link #bufferLock}.
+ */
+ private Staged takeStagedHoldingBufferLock() {
+ byte[] frames;
+ if (summaryData.size() == 0) {
+ frames = bufferData.toByteArray();
+ } else {
+ frames = new byte[bufferData.size() + summaryData.size()];
+ bufferData.copyInto(frames, 0);
+ summaryData.copyInto(frames, bufferData.size());
+ }
+ Staged staged = new Staged(frames, bufferData.size(), bufferedEventCount + bufferedSummaryCount,
+ bufferedSummaryCount);
+ bufferData.reset();
+ summaryData.reset();
+ bufferedEventCount = 0;
+ bufferedSummaryCount = 0;
+ inTransitEvents += staged.events;
+ inTransitSummaries += staged.summaries;
+ inTransitBytes += frames.length;
+ return staged;
+ }
+
+ /**
+ * Puts back what of {@code staged} from {@code from} on never landed, ahead of anything staged since, to
+ * be delivered from memory rather than lost. Requires {@link #bufferLock}.
+ */
+ private void putBackHoldingBufferLock(Staged staged, int from) {
+ byte[] frames = staged.frames;
+ int summariesFrom = Math.max(from, staged.summariesFrom);
+ int events = EventLogFormat.frameCount(frames, from, summariesFrom);
+ int summaries = EventLogFormat.frameCount(frames, summariesFrom, frames.length);
+ inTransitEvents -= events + summaries;
+ inTransitSummaries -= summaries;
+ inTransitBytes -= frames.length - from;
+ prepend(bufferData, frames, from, summariesFrom);
+ prepend(summaryData, frames, summariesFrom, frames.length);
+ bufferedEventCount += events;
+ bufferedSummaryCount += summaries;
+ }
+
+ private static void prepend(ByteArrayOutputStream target, byte[] frames, int from, int to) {
+ if (from == to) {
+ return;
+ }
+ byte[] later = target.toByteArray();
+ target.reset();
+ target.write(frames, from, to - from);
+ writeQuietly(target, later);
+ }
+
+ /**
+ * Hands every staged byte to the kernel, so the events recorded so far survive the process dying.
+ */
+ void commit() {
+ // Answered before ioLock, which a delivery can hold for as long as it takes to read a batch
+ // back: with persistence off there is nothing to write, so nothing worth waiting for.
+ synchronized (bufferLock) {
+ if (!persistEvents) {
+ return;
+ }
+ }
+ synchronized (ioLock) {
+ commitHoldingIoLock();
+ }
+ }
+
+ /**
+ * Writes one serialized event to the log on its own, using up one reservation as
+ * {@link #stageReserved} does, but leaving what anything else has staged to the commit that staged it,
+ * so the caller pays for writing its own bytes and no one else's. It may land ahead of events staged
+ * before it.
+ *
+ * Without persistence, or once it has failed, the event is staged in memory instead, as a commit
+ * leaves it.
+ *
+ * @return false if the event is too large for a frame or for the bytes left in the store
+ */
+ boolean writeReservedAlone(byte[] serializedEvent) {
+ // As in commit: not worth waiting on ioLock for where there is nothing to write.
+ if (!persistEvents) {
+ return stageReserved(serializedEvent);
+ }
+ if (serializedEvent == null || serializedEvent.length == 0
+ || serializedEvent.length > EventLogFormat.MAX_FRAME_SIZE) {
+ releaseReservations(1);
+ return false;
+ }
+ ByteArrayOutputStream framed =
+ new ByteArrayOutputStream(serializedEvent.length + EventLogFormat.FRAME_OVERHEAD);
+ EventLogFormat.writeFrame(framed, serializedEvent);
+ byte[] frame = framed.toByteArray();
+ synchronized (ioLock) {
+ Staged alone;
+ synchronized (bufferLock) {
+ // In the same critical section as the count that replaces it, so the event is never
+ // counted twice or not at all.
+ if (reservedEvents > 0) {
+ reservedEvents--;
+ }
+ if (countedBytesHoldingBufferLock() + frame.length > eventByteCapacity) {
+ return false;
+ }
+ if (!persistEvents) {
+ writeQuietly(bufferData, frame);
+ bufferedEventCount++;
+ return true;
+ }
+ alone = new Staged(frame, frame.length, 1, 0);
+ inTransitEvents++;
+ inTransitBytes += frame.length;
+ }
+ writeHoldingIoLock(alone);
+ }
+ return true;
+ }
+
+ // ---- delivery ----
+
+ /**
+ * Commits, then closes the open log into a batch to deliver, and whatever is still staged in memory
+ * into another.
+ *
+ * Both can hold events at once. Writing that fails after earlier writes succeeded gives up on
+ * persistence with those earlier events in the log and the failed ones back in memory. Closing
+ * only the log would leave the rest for a delivery that reports success without them.
+ *
+ * @return the batches this closed, oldest first: the one from the log, then those from memory; usually
+ * one, and empty when there is nothing to send. A log a commit filled was closed into a batch of its
+ * own as it filled, and is not among them; {@link #pendingBatches()} lists every batch.
+ */
+ List
+ * Called under {@code ioLock} where there is a disk, so that it and the log are closed as one step,
+ * and on its own where there is not.
+ *
+ * @return the batches, oldest first
+ */
+ private List
+ * Requires {@code ioLock} where there is a disk, so that the count is of the listing as it stands.
+ *
+ * @param storedSummaries the summaries this store knows of in the batches the listing found on the disk
+ * @param storedBytes the bytes in the batches the listing found on the disk
+ */
+ private void addHeldBatchesAndRecount(List
+ * Does nothing for a batch this store no longer counts - one {@link #remove} forgot, or a listing
+ * dropped because another process delivered it - since leaving it out would take its events from the
+ * batches still counted.
+ *
+ * A batch on the disk is also locked against expiry, by this process and any other, until
+ * {@link #stoppedSending}: one deleted while its request is out would leave nothing to retry from.
+ */
+ void startedSending(Batch batch) {
+ if (disk == null) {
+ startedSendingHoldingIoLock(batch.payloadId);
+ return;
+ }
+ synchronized (ioLock) {
+ startedSendingHoldingIoLock(batch.payloadId);
+ if (!isHeldInMemory(batch.payloadId)) {
+ disk.startedSending(batch.payloadId, batch.eventCount);
+ }
+ }
+ }
+
+ private boolean isHeldInMemory(String payloadId) {
+ synchronized (heldLock) {
+ return inMemoryBatches.containsKey(payloadId);
+ }
+ }
+
+ /**
+ * Requires {@code ioLock} where there is a disk, so that the batch cannot stop being counted between
+ * finding it counted and leaving it out.
+ */
+ private void startedSendingHoldingIoLock(String payloadId) {
+ synchronized (heldLock) {
+ EventDirectory.Tally counted;
+ HeldBatch held = inMemoryBatches.get(payloadId);
+ if (held != null) {
+ counted = held.tally();
+ } else if (disk != null) {
+ counted = disk.counted(payloadId);
+ } else {
+ counted = null;
+ }
+ if (counted == null) {
+ return;
+ }
+ synchronized (bufferLock) {
+ if (!sendingBatches.containsKey(payloadId)) {
+ sendingBatches.put(payloadId, counted);
+ sendingEvents += counted.events;
+ sendingSummaries += counted.summaries;
+ sendingBytes += counted.bytes;
+ }
+ }
+ }
+ }
+
+ /**
+ * Counts a batch against capacity again, after a request that did not end with it removed. Does nothing
+ * for a batch {@link #remove} already forgot. Ends every {@link #startedSending}, removed or not, so that
+ * the lock against expiry is released either way.
+ */
+ void stoppedSending(Batch batch) {
+ if (disk != null) {
+ synchronized (ioLock) {
+ disk.stoppedSending(batch.payloadId);
+ }
+ }
+ synchronized (bufferLock) {
+ forgetSendingHoldingBufferLock(batch.payloadId);
+ }
+ }
+
+ /**
+ * Requires {@link #bufferLock}.
+ */
+ private void forgetSendingHoldingBufferLock(String payloadId) {
+ EventDirectory.Tally sending = sendingBatches.remove(payloadId);
+ if (sending != null) {
+ sendingEvents -= sending.events;
+ sendingSummaries -= sending.summaries;
+ sendingBytes -= sending.bytes;
+ }
+ }
+
+ /**
+ * Closes every log no live process holds, so their events join the batches to deliver.
+ *
+ * The events in them were recorded by processes that are gone, so there is no one left to add to them.
+ * A log a live process holds, this store's own included, is left alone.
+ *
+ * @return how many events the logs had lost to damage; see {@link EventDirectory.Recovery#lost}
+ */
+ int preparePayloadsFromInterruptedLogs() {
+ if (disk == null) {
+ return 0;
+ }
+ // Each log is read without ioLock, one at a time, and the lock taken only to move it and count it. A
+ // commit at a commit point then waits for at most one rename rather than for every log a crash left to
+ // be read.
+ List
+ * Requires {@code ioLock}.
+ */
+ private void countRecoveredHoldingIoLock(EventDirectory.Tally recovered) {
+ if (recovered.events <= 0) {
+ return;
+ }
+ synchronized (bufferLock) {
+ closedEvents += recovered.events;
+ closedSummaries += recovered.summaries;
+ closedBytes += recovered.bytes;
+ }
+ }
+
+ /**
+ * Deletes stored events older than the maximum age, so that what an application can no longer deliver
+ * does not keep the device's storage, or this store's capacity, forever.
+ *
+ * Its own directory first, under {@code ioLock}; then, once per process, the directories of environments
+ * nothing has written to for as long, outside it, since none of that is this store's.
+ *
+ * @return how many events were discarded, which the caller counts as dropped
+ */
+ int discardExpired() {
+ if (disk == null) {
+ return 0;
+ }
+ long cutoff = System.currentTimeMillis() - maxAgeMillis;
+ int discarded;
+ File directory = null;
+ synchronized (ioLock) {
+ Set
+ * Closes the open log into a batch wherever the next frame would take it past
+ * {@link EventLogFormat#MAX_BATCH_SIZE}, so a commit can land in several batches.
+ */
+ private void commitHoldingIoLock() {
+ Staged staged;
+ synchronized (bufferLock) {
+ if (!persistEvents || bufferedEventCount + bufferedSummaryCount == 0) {
+ // Staged bytes are left where they are: with nowhere durable to put them, memory is
+ // better than dropping them.
+ return;
+ }
+ staged = takeStagedHoldingBufferLock();
+ }
+ writeHoldingIoLock(staged);
+ }
+
+ /** Requires {@code ioLock}, and {@code staged} counted in transit. */
+ private void writeHoldingIoLock(Staged staged) {
+ byte[] bytes = staged.frames;
+
+ int written = 0;
+ try {
+ while (written < bytes.length) {
+ long room;
+ synchronized (bufferLock) {
+ room = EventLogFormat.MAX_BATCH_SIZE - EventLogFormat.FILE_HEADER_SIZE - committedBytes;
+ }
+ if (EventLogFormat.frameSize(bytes, written) > room) {
+ if (!closeFullLogHoldingIoLock()) {
+ break;
+ }
+ continue;
+ }
+ int end = EventLogFormat.endOfFramesFitting(bytes, written, room);
+ int landing = EventLogFormat.frameCount(bytes, written, end);
+ int landingSummaries = staged.summariesIn(written, end);
+ boolean isWhole = written == 0 && end == bytes.length;
+ if (!append(isWhole ? bytes : Arrays.copyOfRange(bytes, written, end))) {
+ break;
+ }
+ // Counted where they landed in the same critical section that stops counting them in
+ // transit, so a capacity check never sees them in both places or in neither.
+ synchronized (bufferLock) {
+ inTransitEvents -= landing;
+ inTransitSummaries -= landingSummaries;
+ inTransitBytes -= end - written;
+ committedEvents += landing;
+ committedSummaries += landingSummaries;
+ committedBytes += end - written;
+ }
+ written = end;
+ }
+ } finally {
+ if (written < bytes.length) {
+ synchronized (bufferLock) {
+ putBackHoldingBufferLock(staged, written);
+ }
+ }
+ }
+ }
+
+ /**
+ * Closes an open log that has no room for the next frame, so that the frame starts a batch of its own.
+ * Requires {@code ioLock}.
+ *
+ * @return whether the next append starts a new log; false if this one could not be closed, which
+ * leaves the frames to be held in memory rather than grow it past the size of a batch
+ */
+ private boolean closeFullLogHoldingIoLock() {
+ int events;
+ int summaries;
+ long bytes;
+ synchronized (bufferLock) {
+ events = committedEvents;
+ summaries = committedSummaries;
+ bytes = committedBytes;
+ }
+ closeOpenLogHoldingIoLock(events, summaries, bytes);
+ synchronized (bufferLock) {
+ return committedBytes == 0;
+ }
+ }
+
+ /**
+ * Requires {@code ioLock}.
+ *
+ * @return false if the bytes could not be written, in which case persistence is given up on
+ */
+ private boolean append(byte[] bytes) {
+ synchronized (bufferLock) {
+ if (!persistEvents) {
+ return false;
+ }
+ }
+ if (!disk.append(bytes)) {
+ synchronized (bufferLock) {
+ persistEvents = false;
+ }
+ return false;
+ }
+ return true;
+ }
+
+ /** Overridable so a test can make writing fail partway through a session, as a full disk does. */
+ @VisibleForTesting
+ FileOutputStream openForAppending(File file) throws IOException {
+ return new FileOutputStream(file, true);
+ }
+
+ @Override
+ public void close() {
+ synchronized (ioLock) {
+ commitHoldingIoLock();
+ if (disk != null) {
+ disk.closeOpenLog();
+ }
+ }
+ }
+
+ /** @return where the events are kept, or null where the application asked for no disk */
+ File getDirectory() {
+ if (disk == null) {
+ return null;
+ }
+ synchronized (ioLock) {
+ return disk.directory();
+ }
+ }
+
+ private static void writeQuietly(ByteArrayOutputStream target, byte[] bytes) {
+ try {
+ target.write(bytes);
+ } catch (IOException e) {
+ // ByteArrayOutputStream.write does not throw; this only satisfies the compiler.
+ }
+ }
+
+ /**
+ * Overridable so a test can make a read fail while the file is intact. Not {@code Files.newInputStream},
+ * which needs API 26.
+ */
+ @SuppressWarnings("IOStreamConstructor")
+ @VisibleForTesting
+ FileInputStream openForReading(File file) throws IOException {
+ return new FileInputStream(file);
+ }
+
+ /**
+ * A group of serialized events closed off for delivery.
+ *
+ * The identifier doubles as the payload ID sent to LaunchDarkly, so a batch retried after the process
+ * died mid-delivery is recognized upstream as the same delivery rather than counted twice.
+ */
+ static final class Batch {
+ final String payloadId;
+ final int eventCount;
+
+ Batch(String payloadId, int eventCount) {
+ this.payloadId = payloadId;
+ this.eventCount = eventCount;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Batch)) {
+ return false;
+ }
+ Batch o = (Batch) other;
+ return payloadId.equals(o.payloadId) && eventCount == o.eventCount;
+ }
+
+ @Override
+ public int hashCode() {
+ return payloadId.hashCode() * 31 + eventCount;
+ }
+
+ @NonNull
+ @Override
+ public String toString() {
+ return "Batch(" + payloadId + ", " + eventCount + " events)";
+ }
+ }
+
+ /**
+ * Every event the store is holding, serialized exactly as it will be sent.
+ *
+ * Reading the log means committing what is staged, so this is not free and is meant for tests and for
+ * diagnosing a store rather than for the recording path.
+ */
+ List
+ * Recording an event does not on its own make it outlive the process. Where
+ * {@link com.launchdarkly.sdk.android.integrations.EventProcessorBuilder#eventPersistence(
+ * com.launchdarkly.sdk.android.integrations.EventPersistence)} is on,
+ * events are written in runs, so one recorded shortly before the process ends may never have been
+ * written at all; this call writes everything recorded so far, and the events then survive whether or
+ * not the delivery does. That write happens on the calling thread before this returns, at
+ * {@link com.launchdarkly.sdk.android.integrations.EventPersistence#DEFERRED} as well as at
+ * {@link com.launchdarkly.sdk.android.integrations.EventPersistence#IMMEDIATE}, so a call from the main
+ * thread touches the disk there. Where persistence is off, events live in memory only and nothing
+ * survives the process, whether or not this was called.
+ *
+ * That is what makes it worth calling where the process is about to end. An application that reports
+ * errors to LaunchDarkly and then crashes should flush from its uncaught exception handler, which runs
+ * while the process is still alive -- {@link #flushAndWait(long, TimeUnit)} is the better choice there,
+ * since it also waits for delivery. Terminations that run no application code, such as {@code SIGKILL},
+ * a native crash, an ANR, or the system reclaiming a backgrounded process, cannot be covered this way,
+ * and they take whatever was recorded since the last write.
*/
void flush();
@@ -152,17 +170,17 @@ public interface LDClientInterface extends Closeable {
*
* Unlike {@link #flush()}, which returns before the events reach the network, this reports
* whether they arrived, which is what makes it usable at a point where the application is about
- * to lose the ability to send them: an uncaught exception handler, a move to the background, or
- * any other last chance. Events buffered in memory do not survive the process, so a caller that
- * knows the process is ending can use this to give them one.
+ * to lose the ability to send them: an uncaught exception handler, or code about to end the
+ * process. Events buffered in memory do not survive the process, so a caller that knows the
+ * process is ending can use this to give them one.
* {@link com.launchdarkly.sdk.android.integrations.LDCrashHandler} does this for
- * uncaught exceptions.
+ * uncaught exceptions. It is for those last chances only, not for routine points such as every
+ * move to the background, where the SDK already flushes; see the paragraph on the backoff below.
*
* It can only help while the process is still running code. An uncaught exception runs its
- * handler first, and a move to the background is announced, so both leave time for this call.
- * A {@code SIGKILL}, an ANR kill, a native crash, and the system reclaiming a backgrounded process
- * run nothing at all, and events still in memory at that moment are lost whatever the
- * application does.
+ * handler first, so it leaves time for this call. A {@code SIGKILL}, an ANR kill, a native crash,
+ * and the system reclaiming a backgrounded process run nothing at all, and events still in memory
+ * at that moment are lost whatever the application does.
*
* The timeout bounds the whole call, including when the SDK is configured for more than one
* environment. Choose it with the caller in mind: a dying process is not a good place to wait on
@@ -174,6 +192,19 @@ public interface LDClientInterface extends Closeable {
* A shorter timeout returns {@code false} before then, so there it bounds the wait rather than
* reporting how the delivery went.
*
+ * Where event persistence is on, this writes down everything recorded so far, as {@link #flush()} does,
+ * but on a thread of the SDK's rather than the calling one, so that the timeout bounds the write as
+ * well: a disk that stalls cannot hold a crashing thread past it. A {@code true} means the events were
+ * written before they were sent. A {@code false} because the timeout expired can come before the write
+ * is done, and the events then survive the process only if it lives long enough for the write to finish.
+ *
+ * It posts even while delivery is backing off after a failure, as {@code close()} does. After a refusal
+ * such as a 401 or 403, the SDK waits about eight and a half minutes before posting again, doubling up to
+ * an hour, and a last chance that sat that out would make no attempt at all. A post that fails then
+ * leaves the wait as it was, so {@link #flush()} and the periodic delivery go on waiting out the same
+ * one rather than a longer one. That also means nothing slows these posts down: a call made routinely,
+ * rather than at a last chance, posts once per call for as long as LaunchDarkly keeps refusing.
+ *
* This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning.
* It is experimental.
*
@@ -182,12 +213,15 @@ public interface LDClientInterface extends Closeable {
* @return true if the events were delivered, or there were none to deliver; false if the timeout
* expired first, the SDK is offline, closed, or otherwise unable to deliver them, or events
* recorded since the last time this was answered were lost on the way, by this delivery or an
- * earlier one. Events the service refuses, even with an error that may pass such as a 503, are
- * retried once straight away and then lost: they are not kept for a later flush, so calling
- * this again does not resend them. A {@code false} because the timeout expired does not mean
- * the events were not sent: the delivery is left running when the caller stops waiting, and
- * may still arrive if the process lives long enough. A caller that resends on {@code false}
- * can therefore cause duplicates.
+ * earlier one. Events the service refuses with an error that may pass, such as a 503, are not
+ * lost: they are retried a second later, then as the backoff allows, and by any later call to
+ * this method, so a later call can still deliver them, and where persistence is on so can a
+ * later run of the application. {@link #flush()} posts nothing while the backoff is waiting.
+ * Events refused with a 401 or 403 are kept too, and posted again once the backoff allows, or by the
+ * next call to this method. A {@code false} because the timeout expired does not
+ * mean the events were not sent: the delivery is left running when the caller stops waiting,
+ * and may still arrive if the process lives long enough. A caller that resends on
+ * {@code false} can therefore cause duplicates.
* @since 5.17.0
*/
boolean flushAndWait(long timeout, TimeUnit unit);
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDUtil.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDUtil.java
index 4a77849b2..01aa42bc2 100644
--- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDUtil.java
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDUtil.java
@@ -174,6 +174,29 @@ static String urlSafeBase64Hash(LDContext context) {
return urlSafeBase64Hash(JsonSerialization.serialize(context));
}
+ /**
+ * A hash for a file or directory name. Unlike {@link #urlSafeBase64Hash(String)} it does not go
+ * through the Android framework's Base64, so it also runs in plain JVM tests, and it has no padding
+ * character.
+ *
+ * @param bytes how many bytes of the digest to render, so a caller can trade length for headroom
+ * @return the first {@code bytes} bytes of the input's SHA-256, in lowercase hex
+ */
+ static String hexDigest(String input, int bytes) {
+ try {
+ byte[] hash = MessageDigest.getInstance("SHA-256")
+ .digest(input.getBytes(Charset.forName("UTF-8")));
+ StringBuilder hex = new StringBuilder(bytes * 2);
+ for (int i = 0; i < bytes; i++) {
+ hex.append(Character.forDigit((hash[i] >> 4) & 0xf, 16));
+ hex.append(Character.forDigit(hash[i] & 0xf, 16));
+ }
+ return hex.toString();
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e); // shouldn't be possible; SHA-256 is built in
+ }
+ }
+
static String urlSafeBase64(LDContext context) {
return Base64.encodeToString(JsonSerialization.serialize(context).getBytes(),
Base64.URL_SAFE + Base64.NO_WRAP);
@@ -239,6 +262,26 @@ static boolean isHttpErrorUnexpected(int statusCode) {
return !isHttpErrorRecoverable(statusCode);
}
+ /**
+ * Tests whether the events service refused an event payload for what it holds rather than for who sent
+ * it: one it cannot read (400), too large for it (413), of a type it does not take (415), or read but
+ * not accepted (422). Posting the same body again will not change the answer, though another body may
+ * be taken.
+ * @param statusCode the HTTP status
+ * @return true for a refusal of the payload
+ */
+ static boolean isEventPayloadRefused(int statusCode) {
+ switch (statusCode) {
+ case 400: // bad request
+ case 413: // content too large
+ case 415: // unsupported media type
+ case 422: // unprocessable content
+ return true;
+ default:
+ return false;
+ }
+ }
+
static void logExceptionAtErrorLevel(LDLogger logger, Throwable ex, String msgFormat, Object... msgArgs) {
logException(logger, ex, true, msgFormat, msgArgs);
}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java
index e1730568a..76c854895 100644
--- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java
@@ -4,26 +4,28 @@
import com.launchdarkly.logging.LogValues;
import com.launchdarkly.sdk.AttributeRef;
import com.launchdarkly.sdk.LDContext;
+import com.launchdarkly.sdk.LDValue;
import com.launchdarkly.sdk.internal.events.AggregatedEventSummarizer;
import com.launchdarkly.sdk.internal.events.Event;
import com.launchdarkly.sdk.internal.events.EventOutputFormatter;
import com.launchdarkly.sdk.internal.events.EventSummarizer;
import com.launchdarkly.sdk.internal.events.EventSummarizerInterface;
import com.launchdarkly.sdk.internal.events.EventsConfiguration;
-import com.launchdarkly.sdk.internal.events.PerContextEventSummarizer;
+import com.launchdarkly.sdk.internal.events.SameContextPerContextEventSummarizer;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
-import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* The counters and the encoder behind {@link DirectEventProcessor}: evaluations are folded into
@@ -41,9 +43,6 @@
*/
final class OutboundEventBuffer {
private static final int INITIAL_OUTPUT_BUFFER_SIZE = 2000;
- private static final Event[] NO_EVENTS = new Event[0];
- private static final byte[] NO_DATA = new byte[0];
- private static final List
+ * Not synchronized: the caller holds its own lock across this and the rest of the recording, as it
+ * does across {@link #takeSummaries}, and a second monitor inside it would only be taken for nothing.
*
* @param event the evaluation
* @return false if this event was not counted, because counting it would have meant holding
* a context beyond the configured capacity
*/
- synchronized boolean summarize(Event.FeatureRequest event) {
+ boolean summarize(Event.FeatureRequest event) {
// Checked here rather than in DirectEventProcessor, for the same reason the sampling ratio
- // is: this is where an event arrives from outside. The processor builds its own through the
- // constructor overload that leaves this false, so a guard there could never fire and would
- // read as dead. java-sdk-internal's DefaultEventProcessor, which this path replaced, honored
- // the flag, and a counter is the one thing no later stage can reconstruct.
+ // is: this is where an event arrives from outside. The processor summarizes from the values
+ // themselves, which have no such flag. java-sdk-internal's DefaultEventProcessor, which this
+ // path replaced, honored it, and a counter is the one thing no later stage can reconstruct.
if (event.isExcludeFromSummaries()) {
return true;
}
+ return summarize(event.getCreationDate(), event.getKey(), event.getVersion(), event.getVariation(),
+ event.getValue(), event.getDefaultVal(), event.getContext());
+ }
+
+ /**
+ * As {@link #summarize(Event.FeatureRequest)}, from the evaluation's values, so that an evaluation
+ * nobody needs a full event for costs no event to be built.
+ */
+ boolean summarize(long timestamp, String flagKey, int flagVersion, int variation, LDValue value,
+ LDValue defaultValue, LDContext context) {
// Only a context that is not being counted yet can be turned away, so reaching the limit costs
// an application evaluating against one context nothing, however many evaluations it does.
- if (countedContexts != null && !countedContexts.contains(event.getContext())) {
- if (countedContexts.size() >= maxContexts) {
- return false;
+ if (countedContexts != null && context != lastCountedContext) {
+ if (!countedContexts.contains(context)) {
+ if (countedContexts.size() >= maxContexts) {
+ return false;
+ }
+ countedContexts.add(context);
}
- countedContexts.add(event.getContext());
+ lastCountedContext = context;
}
- summarizer.summarizeEvent(
- event.getCreationDate(),
- event.getKey(),
- event.getVersion(),
- event.getVariation(),
- event.getValue(),
- event.getDefaultVal(),
- event.getContext()
- );
+ summarizer.summarizeEvent(timestamp, flagKey, flagVersion, variation, value, defaultValue, context);
return true;
}
+ /**
+ * Serializes one event into the bytes it will be sent as, for a caller that keeps events
+ * somewhere other than this buffer's list.
+ *
+ * Serializing at record time rather than at flush is what lets an event be written somewhere that
+ * outlives the process: bytes can be appended to a log, an {@link Event} cannot.
+ *
+ * Deliberately not synchronized. It reads the formatter and writes only locals, so it shares
+ * nothing with a thread recording an event, and taking the lock would make an encode wait on a
+ * counter increment and the other way round.
+ *
+ * Sampling is not applied here. {@link DirectEventProcessor} draws for it once, as the event is
+ * recorded, and drawing again for the same event would square the odds against it.
+ *
+ * @param event the event
+ * @return the event's JSON object, or null if it could not be serialized
+ */
+ byte[] serialize(Event event) {
+ return writeSingleObject(new Event[]{ event }, Collections.
@@ -137,231 +177,174 @@ synchronized boolean summarize(Event.FeatureRequest event) {
* because one that reset the counters alone would spend the cardinality limit on contexts whose
* counters had already gone out, and the limit would never lift.
*
- * Separate from {@link #encode} so that the caller can take the counters in the same critical
- * section it lifts the full events in. An evaluation writes a counter and a full event, and a
- * flush that took the two at different moments could split one evaluation across two payloads.
+ * Separate from {@link #serializeSummaries} so that the caller can take the counters in the same
+ * critical section it lifts the full events in. An evaluation writes a counter and a full event,
+ * and a commit that took the two at different moments could split one evaluation across two
+ * payloads.
+ * Not synchronized, for the reason {@link #summarize(Event.FeatureRequest)} is not.
*/
- synchronized List
- * Deliberately takes no lock. The caller has already taken both the run and the counters, so
- * there is nothing left here to protect, and the encode is the dominant cost on this path --
- * holding a lock across it would make a thread recording an event wait for it. Recording happens
- * on whichever thread evaluated a flag, which on Android is usually the main one.
+ * Takes the counters and serializes them in one call, for a caller with nothing to coordinate the
+ * take with.
*
- * A run that cannot be serialized as a whole is retried per event and per summary. Anything that
- * still fails is dropped and logged; the rest is sent. Everything the encoder can fail on is a
- * property of the data it was handed -- the output stream is a byte array and cannot fail
- * transiently -- so putting a failed item back would only make every later flush fail too.
- *
- * @param run the full events to send, in the order they were recorded
- * @param summaries the counters taken alongside that run
- * @return the payload to send, or null if there was nothing to send; a payload that had to drop
- * something says so through {@link Payload#isComplete()}, and may then hold no events at all
- * @throws IOException if the events could not be serialized
+ * Not for the commit path. That has to take the counters in the same critical section it takes the
+ * pending events in, or a commit can stage an evaluation's counter and leave its full event behind.
*/
- Payload encode(List
+ * Counters live only in memory, so whatever has not been summarized is what a crash takes with
+ * it. Summarizing at points where the events are about to be made durable is what bounds that
+ * loss, and it costs nothing in accuracy: LaunchDarkly sums the counters of every summary it
+ * receives, so several summaries covering the same evaluations count the same as one.
+ *
+ * A summary that cannot be serialized is dropped whole and logged rather than counted back in, and
+ * {@link #takeSerializationFailure} reports it, so the flush that committed it answers false. Losing
+ * an aggregate of evaluations the SDK promised to report is bad, but everything the encoder can
+ * fail on is a property of the data it was handed -- the output stream is a byte array and cannot
+ * fail transiently -- so putting the counters back would make every later flush fail on the same
+ * summary while the counters behind it grew without bound.
+ *
+ * A summary larger than one log frame is not split: the store refuses it whole, and its evaluations
+ * are counted as lost. Reaching that takes one context evaluating dozens of the largest JSON flags
+ * LaunchDarkly allows between two commits.
+ *
+ * Deliberately not synchronized, for the reason {@link #serialize} is not: the counters are taken
+ * under the caller's lock and encoded outside it, so a thread recording an evaluation waits only
+ * for a reference swap and never for the encoder.
+ *
+ * @param summaries the counters taken for this commit
+ * @return one JSON object per summary, empty if there was nothing counted or nothing serialized
+ */
+ List
+ * Each event still becomes its own JSON object, because the store frames them individually and a
+ * frame holding several objects could not be spliced into a payload. What the run shares is the
+ * output stream, the writer and the UTF-8 lookup, which {@link #writeSingleObject} otherwise
+ * allocates on every call.
+ *
+ * Deliberately not synchronized, and not sampling, for the reasons {@link #serialize} is neither.
+ *
+ * @param pending the events, in the order they were recorded
+ * @return one JSON object per event that serialized
*/
- private EncodedPiece tryEncode(Event[] events, List
+ * {@link EventOutputFormatter} only offers to write a whole request body, which is a JSON array,
+ * and the single-event method beside it is private. So the array is written and its brackets are
+ * dropped, which is sound precisely because the count it returns says how many objects are in
+ * there: one object means the bytes between the brackets are that object.
+ *
+ * @return the object's bytes, or null if the formatter did not write exactly one event
+ */
+ private byte[] writeSingleObject(Event[] events, List
+ * This is not the cache directory: the system is free to delete cache files whenever it wants
+ * space, which is the opposite of what data written to survive a crash needs. It is also excluded
+ * from backups, so events recorded on one device are not restored onto another and reported as if
+ * they had happened there.
+ *
+ * @return a directory path
+ */
+ File getNoBackupFilesDir();
}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventPersistence.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventPersistence.java
new file mode 100644
index 000000000..785c7a403
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventPersistence.java
@@ -0,0 +1,60 @@
+package com.launchdarkly.sdk.android.integrations;
+
+/**
+ * How far the SDK goes to make a recorded event outlive the process that recorded it.
+ *
+ * An event lives in memory until it is delivered, so a process that dies before the next flush takes
+ * everything recorded since the last one — including the crash an application was reporting when it died.
+ * Writing events to disk is what closes that gap: they are delivered on a later run instead of being lost.
+ *
+ * {@link #DEFERRED} and {@link #IMMEDIATE} are not stable in this major version, and not subject to any
+ * backwards compatibility guarantees or semantic versioning. They are experimental: their behavior, their
+ * cost on the calling thread, and what they leave on disk may change in a minor release.
+ *
+ * @see EventProcessorBuilder#eventPersistence(EventPersistence)
+ */
+public enum EventPersistence {
+ /**
+ * Events are kept in memory only, and nothing survives the process ending.
+ *
+ * The SDK does not touch the filesystem for events at all. Events that an earlier run with persistence
+ * turned on left on disk are not delivered, and stay there until persistence is turned back on.
+ */
+ DISABLED,
+
+ /**
+ * Events are written to disk, off the thread that recorded them.
+ *
+ * Recording stays as cheap as it is without persistence. Neither {@code track}, {@code identify} nor
+ * evaluating a flag schedules a write of its own: events wait in memory until enough of them have built
+ * up, or until the next flush, delivery, move to the background or close writes them. A process that dies
+ * before then loses them.
+ *
+ * {@code LDClient.flush} is the exception. It writes on the calling thread, as {@link #IMMEDIATE} does,
+ * so that an application ending the process on purpose can rely on it. Moving to the background flushes
+ * too, so it writes the same way. A flush on the main thread with
+ * {@link android.os.StrictMode} detecting disk writes therefore sees a violation. {@code flushAndWait}
+ * writes on a thread of the SDK's instead, so that its timeout bounds the write.
+ *
+ * This level is experimental; see {@link EventPersistence}.
+ */
+ DEFERRED,
+
+ /**
+ * Events are written to disk before {@code track} and {@code identify} return.
+ *
+ * The write happens on the calling thread and covers that one event, costing tens to hundreds of
+ * microseconds depending on the device, however many evaluations are waiting. Those, and the summaries
+ * of them, are written on a thread of the SDK's that the call sets going, so they follow moments later.
+ * In exchange there is no window: a process that dies the instant after {@code track} returns still
+ * reports that event on the next run. Evaluating a flag stays in memory regardless.
+ *
+ * Because this is disk I/O on the calling thread, an application that calls {@code track} or
+ * {@code identify} on the main thread with {@link android.os.StrictMode} detecting disk writes will see
+ * a violation for each call. Choose {@link #DEFERRED} for those callers, which keeps these writes off
+ * the calling thread.
+ *
+ * This level is experimental; see {@link EventPersistence}.
+ */
+ IMMEDIATE
+}
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java
index b0080b17d..fa5d53b8f 100644
--- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java
@@ -29,7 +29,7 @@ public abstract class EventProcessorBuilder implements ComponentConfigurer
* The default value is {@link #DEFAULT_CAPACITY}. A capacity below one is treated as one; to
* stop sending events altogether, use {@link Components#noEvents()} instead.
*
@@ -135,6 +168,73 @@ public EventProcessorBuilder flushIntervalMillis(int flushIntervalMillis) {
return this;
}
+ /**
+ * Sets how far the SDK goes to make a recorded event outlive the process that recorded it.
+ *
+ * Anything other than {@link EventPersistence#DISABLED} appends events to a log under the
+ * application's no-backup files directory and delivers them on a later run, which is what lets the
+ * crash an application was reporting when it died reach LaunchDarkly at all.
+ * {@link EventPersistence#IMMEDIATE} additionally puts the write on the thread that called
+ * {@code track} or {@code identify}, so there is no window in which the event exists only in memory.
+ *
+ * Both levels other than {@link EventPersistence#DISABLED} are experimental, and not subject to any
+ * backwards compatibility guarantees or semantic versioning.
+ *
+ * The default value is {@link #DEFAULT_EVENT_PERSISTENCE}.
+ *
+ * @param eventPersistence how far to go to make events outlive the process
+ * @return the builder
+ */
+ public EventProcessorBuilder eventPersistence(EventPersistence eventPersistence) {
+ this.eventPersistence = eventPersistence == null ? DEFAULT_EVENT_PERSISTENCE : eventPersistence;
+ return this;
+ }
+
+ /**
+ * Sets the maximum number of bytes of events the SDK keeps, on disk and in memory together, whatever
+ * {@link #eventPersistence(EventPersistence)} is.
+ *
+ * It is for every environment the application configures together: with
+ * {@link com.launchdarkly.sdk.android.LDConfig.Builder#secondaryMobileKeys(java.util.Map)}, each
+ * environment gets an equal share of it. That is unlike {@link #capacity(int)}, which each environment
+ * has in full.
+ *
+ * When an environment reaches its share, the SDK discards new events until it successfully delivers the stored ones, as
+ * it does when {@link #capacity(int)} is reached; it never deletes stored events to make room. A fifth
+ * of each share is kept for the summaries of flag evaluations, so that evaluations are still counted after
+ * full events start being discarded.
+ *
+ * Ordinary events reach {@link #capacity(int)} long before this. It binds where events are unusually
+ * large, or where delivery keeps failing and summaries, which {@link #capacity(int)} never refuses,
+ * accumulate. The default value is {@link #DEFAULT_CAPACITY_BYTES}; a value of zero or less means the
+ * default.
+ *
+ * @param capacityBytes the most bytes of events to keep, for all environments together
+ * @return the builder
+ */
+ public EventProcessorBuilder capacityBytes(long capacityBytes) {
+ this.capacityBytes = capacityBytes <= 0 ? DEFAULT_CAPACITY_BYTES : capacityBytes;
+ return this;
+ }
+
+ /**
+ * Sets how long the SDK keeps events it has persisted but not yet delivered, where
+ * {@link #eventPersistence(EventPersistence)} keeps events across launches.
+ *
+ * Events older than this are discarded rather than delivered, and so is everything the SDK stored for
+ * an environment whose mobile key the application stopped using.
+ *
+ * It has no effect where event persistence is {@link EventPersistence#DISABLED}. The default value is
+ * {@link #DEFAULT_MAX_AGE_MILLIS}; a value of zero or less means the default.
+ *
+ * @param maxAgeMillis how long to keep undelivered events, in milliseconds
+ * @return the builder
+ */
+ public EventProcessorBuilder maxAgeMillis(long maxAgeMillis) {
+ this.maxAgeMillis = maxAgeMillis <= 0 ? DEFAULT_MAX_AGE_MILLIS : maxAgeMillis;
+ return this;
+ }
+
/**
* Marks a set of attribute names or subproperties as private.
*
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java
index 98e2bbb26..994e33374 100644
--- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java
@@ -20,7 +20,8 @@
* flush, and those are often the ones that explain it. An uncaught exception runs its handlers while
* the process is still alive, and this one uses that time to call
* {@link LDClientInterface#flushAndWait(long, TimeUnit)} for every configured environment before
- * passing the exception on to the handler that was installed before it.
+ * passing the exception on to the handler that was installed before it. That delivery posts even while the
+ * SDK is backing off after a failed one, as closing the client does.
*
* Install it before your crash reporter:
*
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java
index d958f3e3b..eda21d296 100644
--- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java
@@ -92,8 +92,9 @@ void recordCustomEvent(
* Specifies that any buffered events should be sent as soon as possible, rather than waiting
* for the next flush interval. This method is asynchronous, so events still may not be sent
* until a later time. However, calling {@link Closeable#close()} will synchronously deliver
- * any events that were not yet delivered prior to shutting down, unless the SDK is offline, in
- * which case those events are discarded.
+ * any events that were not yet delivered prior to shutting down, unless the SDK is offline. In
+ * that case nothing is sent; the events are kept for a later run if event persistence is on, and
+ * discarded otherwise.
*/
void flush();
diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizer.java
new file mode 100644
index 000000000..c5498e349
--- /dev/null
+++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizer.java
@@ -0,0 +1,109 @@
+package com.launchdarkly.sdk.internal.events;
+
+import com.launchdarkly.sdk.LDContext;
+import com.launchdarkly.sdk.LDValue;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * {@link PerContextEventSummarizer} with a fast path for the context it saw last, and without the Java 8
+ * library calls that Android lacks below API 24.
+ *
+ * The fast path is the reason this outlives the API-level fix. {@link LDContext#hashCode()} is not cached:
+ * every call flattens and hashes every attribute, so a map keyed on the context costs one full hash per
+ * evaluation, usually on the main thread. An application evaluates against the same {@code LDContext}
+ * instance until it identifies again, so comparing references first skips that hash on nearly every call.
+ *
+ * The API level: java-sdk-internal's summarizer calls {@code Map.computeIfAbsent} with a
+ * {@code java.util.function.Function}, neither of which exists before API 24. An application that has not
+ * turned on core library desugaring gets a {@code NoClassDefFoundError} at the first evaluation it
+ * summarizes, and lint cannot see it because the call is inside a library jar.
+ *
+ * A per-context summary can only be built through {@link EventSummarizer}'s package-private members,
+ * which is why this sits in java-sdk-internal's package rather than the SDK's. Not thread-safe; the
+ * caller serializes every call.
+ */
+public final class SameContextPerContextEventSummarizer implements EventSummarizerInterface {
+ private final Map
+ * +- 4 bytes -+- 2 bytes -+ +- 2 bytes -+- 4 bytes -+- n bytes -+- 4 bytes -+
+ * | LDEV | version | ... | type | size (n) | payload | CRC-32 |
+ * +-----------+-----------+ +-----------+-----------+-----------+-----------+
+ *
+ * The size prefix is what makes a log recoverable: a reader can tell a frame the writer finished
+ * from one it did not, without having to parse the event to find where it ends. The checksum, over
+ * the type, the size and the payload - everything before it in the frame - is what tells a frame the
+ * writer finished from one the storage has since damaged; a reader treats either the same way. The version is what makes an upgrade safe
+ * - a log this version does not understand is discarded rather than misread - and the frame type
+ * leaves room for a later version to write something new into a log this one still reads.
+ *