+ * In-progress client creation: close() waits up to the builder's
+ * {@link QuestDBBuilder#acquireTimeoutMillis(long) acquire timeout},
+ * hard-capped at 5 seconds, for a creation blocked in DNS, TCP, TLS, or a
+ * WebSocket handshake. If that budget expires, the creator retains cleanup
+ * ownership and closes its unpublished client (and releases any SF slot)
+ * when construction returns. This keeps close() bounded even when
+ * {@code connect_timeout} is unset without abandoning late resources.
*
* Outstanding leases: a borrowed {@link Sender} is never torn down
* underneath the thread using it. Instead, close() waits up to the
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index fd16dca7..94018fff 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -248,6 +248,15 @@ public void close() {
}
}
+ /**
+ * Shuts down socket traffic without releasing the descriptor or freeing
+ * buffers that a concurrent I/O worker may still access. The owner must
+ * call {@link #close()} after joining the worker to complete cleanup.
+ */
+ public void closeTraffic() {
+ socket.closeTraffic();
+ }
+
/**
* Connects to a WebSocket server.
*
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java
index b63b4fa9..f6587631 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java
@@ -165,6 +165,10 @@ public class QwpQueryClient implements QuietCloseable {
private final Random failoverRandom = new Random();
private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS;
private String authorizationHeader;
+ // Deterministic lifecycle barrier used by facade shutdown tests. Null in
+ // production; close() clears and invokes it after winning close ownership
+ // without allowing hook failures to prevent resource teardown.
+ private volatile Runnable beforeCloseHook;
// Upper bound (ms) on each TCP connect attempt. 0 (default) falls back to
// the OS connect timeout.
private int connectTimeoutMs = 0;
@@ -620,6 +624,16 @@ public void close() {
// scratch, double-freeing it.
return;
}
+ Runnable hook = beforeCloseHook;
+ beforeCloseHook = null;
+ if (hook != null) {
+ try {
+ hook.run();
+ } catch (Throwable ignored) {
+ // Omit diagnostics for this test-only hook: even rendering its
+ // failure must not prevent production resource cleanup.
+ }
+ }
connected = false;
lastCloseTimedOut = false;
try {
@@ -1004,6 +1018,11 @@ public void seedFailoverRandomForTest(long seed) {
}
}
+ @TestOnly
+ public void setBeforeCloseHookForTest(Runnable hook) {
+ beforeCloseHook = hook;
+ }
+
/**
* Returns true if the most recent {@link #close()} call abandoned the I/O thread
* because it failed to exit within the join timeout. The native buffer pool and
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index d1744065..3f272eb9 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -189,11 +189,19 @@ public class QwpWebSocketSender implements Sender {
// Null in production; set reflectively by tests.
@TestOnly
private volatile java.util.function.Supplier clientFactoryOverride;
+ // Test-only lifecycle witness. drainOnClose() invokes and clears it after
+ // confirming a real unacknowledged close target, immediately before waiting.
+ private volatile Runnable closeDrainWaitingHook;
// close() drain timeout in millis. Default applied at construction.
// 0 or -1 means "fast close" (skip the drain); otherwise close blocks
// up to this many millis for ackedFsn to catch up to publishedFsn.
+ private volatile boolean closeCleanupComplete;
+ private boolean closeCleanupStarted;
private long closeFlushTimeoutMillis = 5_000L;
private volatile boolean closed;
+ // Test-only lifecycle witness. close() invokes and clears it strictly after
+ // publishing closed=true and before starting any drain or teardown work.
+ private volatile Runnable closeStartedHook;
private boolean connected;
private SenderConnectionDispatcher connectionDispatcher;
// Async-delivery sink for SenderConnectionEvent notifications. Default
@@ -278,11 +286,23 @@ public class QwpWebSocketSender implements Sender {
private boolean ownsCursorEngine;
private long pendingBytes;
// Set true by close() once the SF slot flock has been released (the normal
- // teardown path). Stays false if close() bailed early with the I/O thread
- // still running -- then cursorEngine.close() never ran and the flock is
- // still held, so the owning pool MUST keep the slot reserved rather than
- // hand the still-locked dir to the next borrow ("sf slot already in use").
- private boolean slotLockReleased;
+ // teardown path). Stays false if an I/O or manager worker did not stop and
+ // cursorEngine retained the flock, so the owning pool MUST keep the slot
+ // reserved rather than hand the still-locked dir to the next borrow
+ // ("sf slot already in use"). May flip to true LATER via the getter's
+ // re-probe of retainedEngine, once the deferred cleanup (manager-worker
+ // exit path or delegated I/O-thread close) releases the flock — pools
+ // re-probe retired slots to recover their capacity. volatile: written on
+ // the closing thread, read by pool threads.
+ private volatile boolean slotLockReleased;
+ // Optional owning-pool notification, relayed after the engine confirms the
+ // flock release and slotLockReleased is visible to pool re-probes.
+ private volatile Runnable slotLockReleaseListener;
+ // Engine whose close() could not complete during sender close() — its
+ // cleanup is pending on a worker/I/O-thread exit path. isSlotLockReleased()
+ // re-probes it so a late flock release becomes visible to the owning pool.
+ // Only ever set inside close(); null for a sender that closed cleanly.
+ private volatile CursorSendEngine retainedEngine;
private int pendingRowCount;
private SenderProgressDispatcher progressDispatcher;
// Async-delivery sink for ack-watermark advances. Default no-op; a
@@ -1038,13 +1058,26 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) {
* replaying frames get a 2.5s grace window plus a 0.5s stop window
* — worst case ~3s when a drainer sits in a blocking native
* connect (15s background deadline) and must be abandoned to exit
- * on its own.
+ * on its own;
+ *
SF manager stop: normally immediate, bounded by 5s when its worker
+ * is stuck in a filesystem operation. On timeout the slot remains
+ * locked rather than being exposed to a stale worker.
*
*/
@Override
public void close() {
if (!closed) {
closed = true;
+ Runnable hook = closeStartedHook;
+ closeStartedHook = null;
+ if (hook != null) {
+ try {
+ hook.run();
+ } catch (Throwable t) {
+ // A test witness must never prevent production resource cleanup.
+ LOG.error("Error in close-started test hook: {}", String.valueOf(t));
+ }
+ }
boolean ioThreadStopped = true;
// Captures the first error from the flush/drain path AND any
// secondary errors from cleanup steps (added via addSuppressed).
@@ -1175,106 +1208,26 @@ public void close() {
}
if (!ioThreadStopped) {
- // The I/O thread may still be using the socket and microbatch
- // buffers. Freeing them would risk SIGSEGV.
- LOG.error("I/O thread is still running, leaking WebSocket client and microbatch buffers");
- // The engine, however, need not leak: delegate its close to
- // the I/O thread's exit path, which runs it strictly after
- // the thread's last engine access — the mapping and slot
- // lock release as soon as the stuck wire call resolves
- // (bounded by OS timeouts). slotLockReleased intentionally
- // stays false: the lock is released only when the delegated
- // close actually runs, so the pool must not reuse the slot
- // meanwhile. A false return means the thread exited between
- // the failed close() and now — then closing here is safe.
- if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null
- && !cursorSendLoop.delegateEngineClose()) {
- try {
- cursorEngine.close();
- } catch (Throwable t) {
- LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t));
- terminalError = captureCloseError(terminalError, t);
- }
- cursorEngine = null;
- ownsCursorEngine = false;
- slotLockReleased = true;
+ // The worker may still touch every resource below. Hand the
+ // complete sender-owned tail to its exit path rather than
+ // permanently leaking everything except the engine. The
+ // callback is idempotence-gated by closeRemainingResources().
+ if (ownsCursorEngine && cursorEngine != null) {
+ retainedEngine = cursorEngine;
}
- rethrowTerminal(terminalError);
- return;
- }
-
- if (buffer0 != null) {
- try {
- buffer0.close();
- } catch (Throwable t) {
- LOG.error("Error closing buffer0: {}", String.valueOf(t));
- terminalError = captureCloseError(terminalError, t);
- }
- }
- if (buffer1 != null) {
- try {
- buffer1.close();
- } catch (Throwable t) {
- LOG.error("Error closing buffer1: {}", String.valueOf(t));
- terminalError = captureCloseError(terminalError, t);
- }
- }
-
- if (client != null) {
- try {
- client.close();
- } catch (Throwable t) {
- LOG.error("Error closing WebSocket client: {}", String.valueOf(t));
- terminalError = captureCloseError(terminalError, t);
- }
- client = null;
- }
-
- if (ownsCursorEngine && cursorEngine != null) {
- try {
- cursorEngine.close();
- } catch (Throwable t) {
- LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t));
- terminalError = captureCloseError(terminalError, t);
- }
- cursorEngine = null;
- ownsCursorEngine = false;
- }
- // Past the ioThreadStopped guard => cursorEngine.close() ran and
- // released the SF flock in its finally (or this sender owned no
- // engine holding one). Signal the pool it may reuse the slot.
- slotLockReleased = true;
-
- // Shutdown order: dispatcher last, after the I/O loop has stopped
- // producing into it. close() drains pending entries with a short
- // deadline so any final errors land in the user's handler.
- if (errorDispatcher != null) {
- try {
- errorDispatcher.close();
- } catch (Throwable t) {
- LOG.error("Error closing error dispatcher: {}", String.valueOf(t));
- terminalError = captureCloseError(terminalError, t);
- }
- }
- if (progressDispatcher != null) {
- try {
- progressDispatcher.close();
- } catch (Throwable t) {
- LOG.error("Error closing progress dispatcher: {}", String.valueOf(t));
- terminalError = captureCloseError(terminalError, t);
- }
- }
- if (connectionDispatcher != null) {
- try {
- connectionDispatcher.close();
- } catch (Throwable t) {
- LOG.error("Error closing connection dispatcher: {}", String.valueOf(t));
- terminalError = captureCloseError(terminalError, t);
+ Runnable closeCallback = () -> closeRemainingResources(null);
+ if (cursorSendLoop != null && cursorSendLoop.delegateClose(closeCallback)) {
+ rethrowTerminal(terminalError);
+ return;
}
+ // The worker exited between close() failing and delegation.
+ // Cleanup is safe here and its failures remain suppressed on
+ // the original close error.
+ terminalError = closeRemainingResources(terminalError);
+ } else {
+ terminalError = closeRemainingResources(terminalError);
}
- LOG.info("QwpWebSocketSender closed");
-
// If close() ended up holding the same instance the user already
// caught earlier, suppress the rethrow. The user's catch block
// wraps close() (try-with-resources), and Throwable refuses
@@ -1286,14 +1239,57 @@ public void close() {
}
}
+ @TestOnly
+ public boolean isCloseCleanupComplete() {
+ return closeCleanupComplete;
+ }
+
/**
- * True once {@link #close()} has released the store-and-forward slot
- * flock. False means close() leaked the still-running I/O thread (and its
- * resources), so the flock is still held; the owning pool must keep the
- * slot index reserved instead of reusing the still-locked slot dir.
+ * True once the store-and-forward slot flock has been released. False
+ * means an I/O or manager worker did not stop and close() retained the
+ * lock and worker-reachable resources; the owning pool must keep the slot
+ * index reserved instead of reusing the still-locked dir.
+ *
+ * Not a one-shot snapshot: when close() left engine cleanup pending on a
+ * manager-worker quiescence or I/O-thread exit path, this re-probes the
+ * retained engine and latches true the moment that cleanup completes — pools re-probe retired
+ * slots through this getter to recover their capacity. Monotonic:
+ * false→true only, never back. Cheap (volatile reads on every common
+ * path) so pools may call it under their capacity lock; only the rare
+ * orphaned-retry state below does more.
+ *
+ * The probe is also the recovery surface for a retained engine whose
+ * flock-release retry fell off the shared driver because the driver
+ * thread could not start (e.g. OOM at thread creation): close() is
+ * one-shot, so without the re-arm below that slot's capacity would stay
+ * lost until process exit.
*/
public boolean isSlotLockReleased() {
- return slotLockReleased;
+ if (slotLockReleased) {
+ return true;
+ }
+ CursorSendEngine engine = retainedEngine;
+ if (engine != null) {
+ if (engine.isCloseCompleted()) {
+ // Benign latch race: concurrent callers may both observe the
+ // completed cleanup and both write true.
+ slotLockReleased = true;
+ return true;
+ }
+ engine.ensureFlockReleaseRetryScheduled();
+ }
+ return false;
+ }
+
+ /**
+ * Registers a callback for confirmed SF slot-lock release. Pools use this
+ * to wake borrowers without waiting for a timeout or housekeeper tick.
+ */
+ public void setSlotLockReleaseListener(Runnable listener) {
+ slotLockReleaseListener = listener;
+ if (listener != null && slotLockReleased) {
+ listener.run();
+ }
}
@Override
@@ -1709,6 +1705,21 @@ public long getDroppedConnectionNotifications() {
return d == null ? 0L : d.getDroppedNotifications();
}
+ @TestOnly
+ public SenderConnectionDispatcher getConnectionDispatcherForTesting() {
+ return connectionDispatcher;
+ }
+
+ @TestOnly
+ public SenderErrorDispatcher getErrorDispatcherForTesting() {
+ return errorDispatcher;
+ }
+
+ @TestOnly
+ public SenderProgressDispatcher getProgressDispatcherForTesting() {
+ return progressDispatcher;
+ }
+
/**
* Number of {@link SenderError} notifications dropped because the
* bounded inbox was full. Non-zero means the user-supplied
@@ -2135,6 +2146,31 @@ public void setClientFactoryOverride(java.util.function.Suppliercancel() could closeTraffic() a client the walk no longer owns;
+ // proven a no-op on both production transports today (closed-fd closeTraffic
+ // no-ops), but a future custom transport that threw on a closed socket would
+ // spuriously loud-fail close(). Null-guarded: the no-arg reconnect() path
+ // and the Unsafe.allocateInstance bare-loop tests pass a null handle. Pairs
+ // with the success-path clear() after upgrade().
+ private static void clearInFlight(CursorWebSocketSendLoop.ConnectCancellation cancellation) {
+ if (cancellation != null) {
+ cancellation.clear();
+ }
+ }
+
+ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLoop.ConnectCancellation cancellation) {
// Background (drainer) factories share this connect walk -- endpoint
// list and hostTracker HEALTH state (never the shared round: a
// background sweep walks its own RoundCursor and records with
@@ -2808,9 +2886,34 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) {
newClient.setQwpClientId(QwpConstants.CLIENT_ID);
newClient.setQwpRequestDurableAck(requestDurableAck);
newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs));
+ if (cancellation != null) {
+ // Publish the client we are about to block on so a
+ // concurrent CursorWebSocketSendLoop.close() can break its
+ // traffic and unwind a black-holed native connect
+ // (connect_timeout=0 => OS SYN-retry) instead of hanging on
+ // the untimed shutdown-latch await. The publish-then-check
+ // handshake pairs with ConnectCancellation.cancel(): if we
+ // observe cancellation here we skip the blocking connect
+ // entirely; otherwise cancel() observed this client and
+ // breaks it. The walk's per-attempt catch disposes the
+ // client and, since running has flipped false, the
+ // top-of-loop ctx.isAborted() gate ends the walk.
+ cancellation.publish(newClient);
+ if (cancellation.isCancelled()) {
+ throw new LineSenderException(ctx.abortMessage());
+ }
+ }
newClient.connect(ep.host, ep.port);
int upgradeTimeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE);
newClient.upgrade(WRITE_PATH, upgradeTimeoutMs, authorizationHeader);
+ if (cancellation != null) {
+ // connect()+upgrade() completed: this client is no longer
+ // blocking, so drop it from the in-flight handle before it
+ // becomes the loop's `client` field. close() must then break
+ // its traffic via the field path exactly once -- leaving it
+ // in the handle too would double-shut-down the socket.
+ cancellation.clear();
+ }
} catch (HttpClientException e) {
// Close BEFORE classify: the sibling catch (Error) below does not
// guard catch-arm bodies, so an Error thrown inside classify()
@@ -2820,6 +2923,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) {
// upgradeStatusCode) that are set during upgrade() and survive
// close().
newClient.close();
+ clearInFlight(cancellation);
HttpClientException classified = QwpUpgradeFailures.classify(newClient, ep.host, ep.port, e);
if (classified instanceof QwpIngressRoleRejectedException) {
QwpIngressRoleRejectedException re = (QwpIngressRoleRejectedException) classified;
@@ -2866,6 +2970,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) {
continue;
} catch (Exception e) {
newClient.close();
+ clearInFlight(cancellation);
hostTracker.recordTransportError(idx, !background);
lastError = e;
if (!background) {
@@ -2888,6 +2993,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) {
// the cursor reconnect loop, BackgroundDrainer) rethrows Error
// rather than retrying, so this stays a loud one-shot failure.
closeQuietlyOnError(newClient);
+ clearInFlight(cancellation);
throw e;
}
// Guard the post-upgrade tail: from here until newClient is
@@ -3084,6 +3190,83 @@ private void checkTableSelected() {
}
}
+ private synchronized Throwable closeRemainingResources(Throwable terminalError) {
+ if (closeCleanupStarted) {
+ return terminalError;
+ }
+ closeCleanupStarted = true;
+ try {
+ try {
+ buffer0.close();
+ } catch (Throwable t) {
+ LOG.error("Error closing buffer0: {}", String.valueOf(t));
+ terminalError = captureCloseError(terminalError, t);
+ }
+ try {
+ buffer1.close();
+ } catch (Throwable t) {
+ LOG.error("Error closing buffer1: {}", String.valueOf(t));
+ terminalError = captureCloseError(terminalError, t);
+ }
+ if (client != null) {
+ try {
+ client.close();
+ } catch (Throwable t) {
+ LOG.error("Error closing WebSocket client: {}", String.valueOf(t));
+ terminalError = captureCloseError(terminalError, t);
+ }
+ client = null;
+ }
+ if (ownsCursorEngine && cursorEngine != null) {
+ CursorSendEngine engine = cursorEngine;
+ try {
+ engine.close();
+ } catch (Throwable t) {
+ LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t));
+ terminalError = captureCloseError(terminalError, t);
+ }
+ if (engine.isCloseCompleted()) {
+ cursorEngine = null;
+ ownsCursorEngine = false;
+ slotLockReleased = true;
+ } else {
+ slotLockReleased = false;
+ retainedEngine = engine;
+ }
+ } else {
+ slotLockReleased = true;
+ }
+ if (errorDispatcher != null) {
+ try {
+ errorDispatcher.close();
+ } catch (Throwable t) {
+ LOG.error("Error closing error dispatcher: {}", String.valueOf(t));
+ terminalError = captureCloseError(terminalError, t);
+ }
+ }
+ if (progressDispatcher != null) {
+ try {
+ progressDispatcher.close();
+ } catch (Throwable t) {
+ LOG.error("Error closing progress dispatcher: {}", String.valueOf(t));
+ terminalError = captureCloseError(terminalError, t);
+ }
+ }
+ if (connectionDispatcher != null) {
+ try {
+ connectionDispatcher.close();
+ } catch (Throwable t) {
+ LOG.error("Error closing connection dispatcher: {}", String.valueOf(t));
+ terminalError = captureCloseError(terminalError, t);
+ }
+ }
+ LOG.info("QwpWebSocketSender closed");
+ return terminalError;
+ } finally {
+ closeCleanupComplete = true;
+ }
+ }
+
private int countNonEmptyTables(ObjList keys) {
int tableCount = 0;
for (int i = 0, n = keys.size(); i < n; i++) {
@@ -3185,6 +3368,16 @@ private void drainOnClose(boolean errorOwnedByCustomHandler) {
if (cursorEngine.ackedFsn() >= target) {
return;
}
+ Runnable hook = closeDrainWaitingHook;
+ closeDrainWaitingHook = null;
+ if (hook != null) {
+ try {
+ hook.run();
+ } catch (Throwable t) {
+ // A test witness must never prevent production resource cleanup.
+ LOG.error("Error in close-drain-waiting test hook: {}", String.valueOf(t));
+ }
+ }
long deadlineNanos = System.nanoTime() + closeFlushTimeoutMillis * 1_000_000L;
while (cursorEngine.ackedFsn() < target) {
// Stop on a latched terminal (acks will never reach target);
@@ -3545,6 +3738,14 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm
resetTableBuffersAfterFlush(keys);
}
+ private void onSlotLockReleased() {
+ slotLockReleased = true;
+ Runnable listener = slotLockReleaseListener;
+ if (listener != null) {
+ listener.run();
+ }
+ }
+
private void resetTableBuffersAfterFlush(ObjList keys) {
for (int i = 0, n = keys.size(); i < n; i++) {
CharSequence tableName = keys.getQuick(i);
@@ -3850,7 +4051,12 @@ boolean isAborted() {
@Override
public WebSocketClient reconnect() {
- return buildAndConnect(this);
+ return buildAndConnect(this, null);
+ }
+
+ @Override
+ public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) {
+ return buildAndConnect(this, cancellation);
}
}
}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java
index 469c91ea..c1e6c96a 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java
@@ -24,7 +24,9 @@
package io.questdb.client.cutlass.qwp.client.sf.cursor;
+import io.questdb.client.std.Crc32c;
import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
import io.questdb.client.std.MemoryTag;
import io.questdb.client.std.QuietCloseable;
import io.questdb.client.std.Unsafe;
@@ -42,76 +44,72 @@
* "everything <= N is durable"), so a single monotonic watermark suffices;
* no per-frame bitmap is needed.
*
- * Layout (16 bytes, little-endian, mmap'd for the engine's lifetime):
+ * Layout (128 bytes, little-endian, mmap'd for the engine's lifetime):
+ * two independently CRC-protected 64-byte records. Each record contains:
*
+ * {@link #write(long)} rewrites the record not selected by the current
+ * generation and stores its CRC last. Recovery selects the valid record with
+ * the greatest generation. A torn update therefore falls back to the older
+ * valid record; if neither record validates, recovery conservatively uses the
+ * segment-derived seed.
*
- * Zero-alloc, zero-syscall writes: {@link #open(String)} opens the
- * file and maps the 16 bytes once. {@link #write(long)} is a single
- * 8-byte aligned {@code Unsafe.putLong} into the mapped region. No
- * malloc/free, no read/write syscalls, no rename, on the manager's hot
- * tick path. An 8-byte aligned store is hardware-atomic on x86_64 and
- * arm64, and disk-atomic within one sector (the file is 16 bytes —
- * trivially within one sector), so a torn FSN across a crash boundary
- * is not a concern.
+ * Zero-alloc, store-only ACK writes: {@link #open(String)} maps both
+ * records once. Ordinary ACK-only manager updates mutate the inactive record
+ * in the mapping and require no malloc/free, read/write syscalls, or rename.
*
- * Why no CRC: the watermark is an optimization. If the on-disk
- * value is somehow corrupted (filesystem bug, hardware fault), the
- * recovery path's {@code max(lowestBase - 1, watermark)} clamp absorbs
- * the inconsistency: a stale-low watermark just means more re-replay, a
- * stale-high watermark is impossible because no write produces an FSN
- * higher than the segments on disk could account for. CRC adds
- * complexity (multi-store with fence + read-side validate) for
- * marginal additional safety.
+ * fsync cadence: ordinary ACK-only manager updates call
+ * {@link #write(long)} and stay syscall-free. Each non-empty background disk-trim
+ * quantum calls {@link #sync()} once (one mmap msync and one fd fsync), fsyncs
+ * the slot directory before unlinking, and fsyncs it again after the batch. A
+ * fully drained close uses the same covering order so the durable watermark
+ * guards any acknowledged segment that a host crash restores.
*
- * fsync cadence: intentionally NOT performed. Host crash falls
- * back to recovery's {@code lowestBase - 1} seed (same as before this
- * feature, no regression), at the cost of a bounded re-replay window
- * for whatever durable-acks landed since the last successful page
- * cache flush.
- *
- * Lifecycle: single-writer (the {@link SegmentManager} worker
- * thread) after construction. Read once at engine startup (any thread,
- * before the manager observes the entry). Close releases the mapping
- * and fd. Not thread-safe for concurrent writers.
+ * Lifecycle: single-writer (the {@link SegmentManager} worker thread)
+ * after construction. Read once at engine startup (any thread, before the
+ * manager observes the entry). Close releases the mapping and fd. Not
+ * thread-safe for concurrent writers.
*/
public final class AckWatermark implements QuietCloseable {
/**
- * Filename of the watermark within the slot directory.
- * Dot-prefixed so directory enumerators that filter by extension
- * (segment recovery, OrphanScanner) skip it automatically.
+ * Filename of the watermark within the slot directory. Dot-prefixed so
+ * directory enumerators that filter by extension skip it automatically.
*/
public static final String FILE_NAME = ".ack-watermark";
- public static final int FILE_SIZE = 16;
+ public static final int FILE_SIZE = 128;
/**
- * Sentinel returned by {@link #read()} when the watermark file is
- * present but has not yet been written to (the magic field is zero
- * because the OS zero-filled the freshly created file).
+ * Sentinel returned by {@link #read()} when neither watermark record is
+ * valid.
*/
public static final long INVALID = Long.MIN_VALUE;
static final int FILE_MAGIC = 0x31574B41; // 'AKW1' little-endian
- private static final int FSN_OFFSET = 8;
+ private static final int CRC_OFFSET = 60;
+ private static final int FSN_OFFSET = 16;
private static final Logger LOG = LoggerFactory.getLogger(AckWatermark.class);
private static final int MAGIC_OFFSET = 0;
+ private static final int RECORD_SIZE = 64;
+ private static final int VERSION = 1;
private final int fd;
+ private final FilesFacade filesFacade;
private final long mmapAddress;
private boolean closed;
- // Stamped once per process either at open() (if the file already
- // has the magic from a prior session) or on the first write() that
- // observes it unset. After the flag flips, write() degenerates to
- // a single 8-byte putLong against the mapped FSN slot -- no memory
- // load of magic on the hot path. Manager-thread-only after
- // construction; no synchronisation needed.
- private boolean magicWritten;
-
- private AckWatermark(int fd, long mmapAddress, boolean magicAlreadyWritten) {
+ private long fsn;
+ private long generation;
+
+ private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress,
+ long generation, long fsn) {
this.fd = fd;
+ this.filesFacade = filesFacade;
+ this.fsn = fsn;
+ this.generation = generation;
this.mmapAddress = mmapAddress;
- this.magicWritten = magicAlreadyWritten;
}
@Override
@@ -122,111 +120,163 @@ public void close() {
Files.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT);
}
if (fd >= 0) {
- Files.close(fd);
+ filesFacade.close(fd);
}
}
/**
- * Opens (creating if absent) the watermark file in {@code slotDir}
- * and maps it for the engine's lifetime. Returns {@code null} on
- * any setup failure (open fail, mmap fail, unexpected size) — the
- * caller falls back to the no-watermark behaviour, no exception
- * escapes. Idempotent at the engine layer: a stale file from a
- * prior session is reused as-is; the first {@link #write(long)}
- * stamps the magic and the new FSN atomically.
+ * Opens (creating if absent) the watermark file in {@code slotDir} and
+ * maps it for the engine's lifetime. Returns {@code null} on any setup
+ * failure, leaving the caller to choose whether its durability contract
+ * permits operation without one.
+ *
+ * Wrong-sized files, including the legacy 16-byte non-CRC format, are
+ * reset. Trusting a legacy FSN would retain the torn-write ambiguity this
+ * format removes; resetting it causes conservative replay from the
+ * segment-derived seed instead.
*/
public static AckWatermark open(String slotDir) {
+ return open(FilesFacade.INSTANCE, slotDir);
+ }
+
+ static AckWatermark open(FilesFacade filesFacade, String slotDir) {
String filePath = slotDir + "/" + FILE_NAME;
- // Decide by size: existing-and-correct -> openRW preserves the
- // previous session's watermark (defeating which is the whole
- // point of NOT calling openCleanRW unconditionally); missing or
- // wrong-sized -> openCleanRW + allocate creates a fresh
- // FILE_SIZE-byte file (zero magic, read() reports INVALID until
- // the first write).
- long existing = Files.exists(filePath) ? Files.length(filePath) : -1L;
+ long existing = filesFacade.exists(filePath) ? filesFacade.length(filePath) : -1L;
int fd;
if (existing == FILE_SIZE) {
- fd = Files.openRW(filePath);
+ fd = filesFacade.openRW(filePath);
} else {
- fd = Files.openCleanRW(filePath);
- if (fd >= 0 && !Files.allocate(fd, FILE_SIZE)) {
- // FilesFacade.allocate contract on a false return:
- // close the fd AND unlink the partial file.
- Files.close(fd);
- Files.remove(filePath);
+ fd = filesFacade.openCleanRW(filePath);
+ if (fd >= 0 && !filesFacade.allocate(fd, FILE_SIZE)) {
+ // FilesFacade.allocate contract on a false return: close the
+ // fd and unlink the partial file.
+ filesFacade.close(fd);
+ filesFacade.remove(filePath);
fd = -1;
}
}
if (fd < 0) {
- LOG.warn("ack watermark {} could not be opened (rc={}); proceeding without it",
- filePath, fd);
+ LOG.warn("ack watermark {} could not be opened (rc={})", filePath, fd);
return null;
}
long addr = Files.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
if (addr == Files.FAILED_MMAP_ADDRESS) {
- LOG.warn("ack watermark {} could not be mmapped; proceeding without it", filePath);
- Files.close(fd);
+ LOG.warn("ack watermark {} could not be mmapped", filePath);
+ filesFacade.close(fd);
return null;
}
- // Inspect the existing magic once at open time. If it's already
- // set (cross-session reopen, e.g. recovery after a clean
- // shutdown), the first write() can skip the magic store
- // entirely and degenerate to a single 8-byte FSN put.
- int magic = Unsafe.getUnsafe().getInt(addr + MAGIC_OFFSET);
- return new AckWatermark(fd, addr, magic == FILE_MAGIC);
+ Record selected = selectRecord(addr);
+ return selected == null
+ ? new AckWatermark(filesFacade, fd, addr, 0L, INVALID)
+ : new AckWatermark(filesFacade, fd, addr, selected.generation, selected.fsn);
}
/**
- * Best-effort removal of a stale watermark file. Used by the
- * engine startup path when no segments are recovered — a stale
- * watermark file with no segments behind it is meaningless and
- * would only confuse the next session's seed.
+ * Best-effort removal of a stale watermark file. Used by the engine
+ * startup path when no segments are recovered.
*/
public static void removeOrphan(String slotDir) {
- Files.remove(slotDir + "/" + FILE_NAME);
+ removeOrphan(FilesFacade.INSTANCE, slotDir);
+ }
+
+ static boolean removeOrphan(FilesFacade filesFacade, String slotDir) {
+ return filesFacade.remove(slotDir + "/" + FILE_NAME);
}
/**
- * Single-load read of the current FSN. Returns {@link #INVALID} if
- * the file has never been written (magic field is zero, i.e. the
- * file was freshly created by {@link #open(String)} and no
- * {@link #write(long)} has run yet against this slot).
+ * Returns the FSN from the greatest-generation valid record, or
+ * {@link #INVALID} when neither record validates.
*/
public long read() {
if (closed) return INVALID;
- int magic = Unsafe.getUnsafe().getInt(mmapAddress + MAGIC_OFFSET);
- if (magic != FILE_MAGIC) {
- // Either freshly created (all zeros) or some kind of corruption.
- // Either way, fall back to the segment-derived seed.
- return INVALID;
+ Record selected = selectRecord(mmapAddress);
+ if (selected == null) {
+ fsn = INVALID;
+ generation = 0L;
+ } else {
+ fsn = selected.fsn;
+ generation = selected.generation;
+ }
+ return fsn;
+ }
+
+ /**
+ * Flushes the mapped bytes and then the backing fd. The caller must sync
+ * the slot directory after this succeeds so a newly-created watermark's
+ * directory entry is durable before segment deletion begins.
+ */
+ public void sync() {
+ if (closed) {
+ throw new IllegalStateException("ack watermark is closed");
+ }
+ if (filesFacade.msync(mmapAddress, FILE_SIZE, false) != 0) {
+ throw new IllegalStateException("could not msync ack watermark");
+ }
+ if (filesFacade.fsync(fd) != 0) {
+ throw new IllegalStateException("could not fsync ack watermark");
}
- return Unsafe.getUnsafe().getLong(mmapAddress + FSN_OFFSET);
}
/**
- * Atomically updates the persisted FSN. Single 8-byte aligned
- * store to the mapped region. The first write also stamps the
- * magic so the next session's {@link #read()} can distinguish
- * "valid watermark" from "freshly created file".
+ * Updates the inactive record and selects it in memory only after its CRC
+ * has been stored. The next {@link #sync()} makes the complete update
+ * durable before any covered segment is deleted.
*
- * Caller responsibility: monotonic ordering. The manager's tick
- * loop should only call this when {@code fsn} has advanced past
- * the last write.
+ * Caller responsibility: monotonic ordering. The manager's tick loop only
+ * calls this when {@code fsn} has advanced past the last write.
*/
public void write(long fsn) {
if (closed) return;
- // Steady-state hot path: a single 8-byte aligned putLong, no
- // memory load of the magic. Order matters only on the very
- // first write of a fresh file: FSN first so the next reader
- // that observes the magic (set immediately below) also
- // observes a valid FSN -- no fence needed because the same
- // thread reads both in program order, and crash recovery
- // resumes a fresh process that sees whatever disk-level state
- // the kernel flushed.
- Unsafe.getUnsafe().putLong(mmapAddress + FSN_OFFSET, fsn);
- if (!magicWritten) {
- Unsafe.getUnsafe().putInt(mmapAddress + MAGIC_OFFSET, FILE_MAGIC);
- magicWritten = true;
+ long nextGeneration = generation + 1L;
+ long recordAddress = mmapAddress + (nextGeneration & 1L) * RECORD_SIZE;
+ Unsafe.getUnsafe().setMemory(recordAddress, RECORD_SIZE, (byte) 0);
+ Unsafe.getUnsafe().putInt(recordAddress + MAGIC_OFFSET, FILE_MAGIC);
+ Unsafe.getUnsafe().putInt(recordAddress + 4, VERSION);
+ Unsafe.getUnsafe().putLong(recordAddress + 8, nextGeneration);
+ Unsafe.getUnsafe().putLong(recordAddress + FSN_OFFSET, fsn);
+ int crc = Crc32c.update(Crc32c.INIT, recordAddress, CRC_OFFSET);
+ Unsafe.getUnsafe().putInt(recordAddress + CRC_OFFSET, crc);
+ generation = nextGeneration;
+ this.fsn = fsn;
+ }
+
+ private static Record readRecord(long address) {
+ if (Unsafe.getUnsafe().getInt(address + MAGIC_OFFSET) != FILE_MAGIC
+ || Unsafe.getUnsafe().getInt(address + 4) != VERSION) {
+ return null;
+ }
+ int expectedCrc = Unsafe.getUnsafe().getInt(address + CRC_OFFSET);
+ int actualCrc = Crc32c.update(Crc32c.INIT, address, CRC_OFFSET);
+ if (expectedCrc != actualCrc) {
+ return null;
+ }
+ long generation = Unsafe.getUnsafe().getLong(address + 8);
+ long fsn = Unsafe.getUnsafe().getLong(address + FSN_OFFSET);
+ if (generation <= 0 || fsn < -1L) {
+ return null;
+ }
+ return new Record(generation, fsn);
+ }
+
+ private static Record selectRecord(long address) {
+ Record first = readRecord(address);
+ Record second = readRecord(address + RECORD_SIZE);
+ if (first == null) {
+ return second;
+ }
+ if (second == null || first.generation > second.generation) {
+ return first;
+ }
+ return second;
+ }
+
+ private static final class Record {
+ private final long fsn;
+ private final long generation;
+
+ private Record(long generation, long fsn) {
+ this.fsn = fsn;
+ this.generation = generation;
}
}
}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
index 64ca75d0..90c227fa 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
@@ -26,10 +26,17 @@
import io.questdb.client.std.Compat;
import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
import io.questdb.client.std.ObjList;
import io.questdb.client.std.QuietCloseable;
+import org.jetbrains.annotations.TestOnly;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.LockSupport;
+import java.util.function.LongConsumer;
/**
* Facade that bundles a {@link SegmentRing} with a {@link SegmentManager} and
@@ -63,6 +70,18 @@ public final class CursorSendEngine implements QuietCloseable {
public static final long DEFAULT_APPEND_DEADLINE_NANOS = 30_000_000_000L;
private static final org.slf4j.Logger LOG =
org.slf4j.LoggerFactory.getLogger(CursorSendEngine.class);
+ private static final ThreadFactory DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY =
+ runnable -> new Thread(runnable, "qdb-sf-flock-release-retry");
+ private static final long FLOCK_RELEASE_RETRY_BASE_PARK_NANOS = 100_000_000L; // 100 ms
+ private static final Object FLOCK_RELEASE_RETRY_LOCK = new Object();
+ private static final long FLOCK_RELEASE_RETRY_MAX_PARK_NANOS = 5_000_000_000L; // 5 s
+ private static final ArrayDeque FLOCK_RELEASE_RETRY_QUEUE = new ArrayDeque<>();
+ private static volatile Runnable afterFlockReleaseRetryFailureHook;
+ private static volatile Runnable beforeDeferredCloseCreationHook;
+ private static volatile LongConsumer flockReleaseRetryParkOverride;
+ private static Thread flockReleaseRetryThread;
+ private static volatile ThreadFactory flockReleaseRetryThreadFactory =
+ DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY;
private final long appendDeadlineNanos;
// Number of times appendBlocking observed BACKPRESSURE_NO_SPARE on its first
// ring.appendOrFsn attempt. One increment per blocking-call that had to wait
@@ -70,6 +89,11 @@ public final class CursorSendEngine implements QuietCloseable {
// writer; volatile because the user may sample it from any thread.
private final java.util.concurrent.atomic.AtomicLong backpressureStallCount =
new java.util.concurrent.atomic.AtomicLong();
+ // Constructed before an owned manager acquires its native path scratch, so
+ // callback allocation failure cannot orphan manager resources. A timed-out
+ // close can then hand it to either manager path without allocating.
+ private final Runnable deferredClose;
+ private final FilesFacade filesFacade;
private final SegmentManager manager;
// We own the manager iff the user constructed us with no manager — in that
// case close() also stops the manager. When the manager is shared across
@@ -104,16 +128,67 @@ public final class CursorSendEngine implements QuietCloseable {
// range with a cumulative self-acknowledge once everything below is
// server-acked (CursorWebSocketSendLoop.tryRetireOrphanTail).
private long recoveredOrphanTipFsn = -1L;
- // Engine-owned mmap'd watermark file. {@code null} in memory mode and
- // in disk mode if open() failed (we proceed without it; recovery just
- // falls back to lowestBase - 1). Lifetime tied to the engine: opened
- // in the constructor, closed by {@link #close()}. The segment manager
- // writes through this on every tick where ackedFsn has advanced.
+ // Engine-owned mmap'd watermark file. {@code null} only in memory mode;
+ // disk mode fails construction unless the watermark is usable because
+ // segment-derived recovery cannot distinguish acknowledged residue from
+ // replayable frames. Lifetime tied to the engine: opened in the
+ // constructor, closed by {@link #close()}. The segment manager writes
+ // through this on every tick where ackedFsn has advanced.
private final AckWatermark watermark;
// close() is publicly callable from any thread (Sender.close from a user
// thread, JVM shutdown hooks, test cleanup). volatile + synchronized
// close() makes the check-and-set atomic and gives readers a fence.
private volatile boolean closed;
+ // True once close() has run its full cleanup sequence INCLUDING a
+ // CONFIRMED slot-flock release — finishClose() publishes this strictly
+ // after SlotLock.release() reports success, never before. Pool threads
+ // treat the flip as "the slot dir is reusable" and free the slot index
+ // the moment they observe it (QwpWebSocketSender.isSlotLockReleased ->
+ // SenderPool.reprobeRetiredSlots), so publishing before the release
+ // would let a replacement engine's SlotLock.acquire collide with the
+ // still-open fd. Stays false when a close attempt could not confirm
+ // manager-worker quiescence (or the flock release itself failed) and had
+ // to leak the ring/watermark/slot lock — in that case a later close()
+ // call retries the cleanup (the worker may have exited by then).
+ // volatile: latched by finishClose(), but read lock-free by
+ // isCloseCompleted() from pool threads re-probing a retired slot (see the
+ // getter for why it must not synchronize).
+ private volatile boolean closeCompleted;
+ // Invoked after closeCompleted publishes a confirmed flock release. Used
+ // by an owning sender pool to wake capacity-starved borrowers immediately.
+ private volatile Runnable slotLockReleaseListener;
+ // Test-only hook run by finishClose() between the terminal cleanup and
+ // the flock release. Lets a test park the releasing thread inside the
+ // cleanup/release window and assert that closeCompleted stays false —
+ // i.e. that completion is never observable while the flock is still
+ // held. volatile: finishClose may run on the manager worker's exit
+ // thread while the hook is installed from a test thread.
+ private volatile Runnable beforeFlockReleaseHook;
+ // Ensures this engine has at most one entry in the shared flock-release
+ // retry driver. The error path only: ordinary closes never enqueue work.
+ private final AtomicBoolean flockReleaseRetryStarted = new AtomicBoolean();
+ // Published before deferredClose is registered. The manager lock provides
+ // the callback handoff fence; volatile also covers a direct test/retry read.
+ private volatile boolean fullyDrainedForDeferredClose;
+ // Exactly-once claim on the terminal cleanup (finishClose). Contended by
+ // close() and a worker-exit handoff (completeDeferredClose); whoever wins
+ // the CAS runs the cleanup, the loser never touches ring/watermark/flock.
+ // Deliberately NOT the engine monitor: a retried close() holds the
+ // monitor while joining the manager worker, and the worker cannot die
+ // until its exit-path cleanup finishes — monitor-based exclusion would
+ // stall that close() for its full join budget (test-visible livelock).
+ // With the CAS the worker's cleanup never blocks, so the join returns as
+ // soon as the pass ends.
+ private final AtomicBoolean terminalCleanupClaimed = new AtomicBoolean();
+ // True only after a quiescent close reached the final watermark barrier
+ // and that barrier failed. It permits the shared retry driver to claim
+ // terminal cleanup; false while cleanup is merely waiting for manager
+ // quiescence, where an independent retry would race the live worker.
+ private volatile boolean terminalCleanupRetryReady;
+ // Published only after ring/watermark/unlink cleanup is finished. A close
+ // that loses terminalCleanupClaimed may retry the flock only after this
+ // becomes true, otherwise it could expose the slot while cleanup is live.
+ private volatile boolean terminalResourcesCleaned;
// Producer-thread-only: timestamp of the last "we're backpressured" log
// line, used to throttle. Plain long is fine.
private long lastBackpressureLogNs;
@@ -137,9 +212,7 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes) {
*/
public CursorSendEngine(String sfDir, long segmentSizeBytes,
long maxTotalBytes, long appendDeadlineNanos) {
- this(sfDir, segmentSizeBytes,
- new SegmentManager(segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes),
- true, appendDeadlineNanos);
+ this(sfDir, segmentSizeBytes, null, true, maxTotalBytes, appendDeadlineNanos);
}
/**
@@ -153,6 +226,22 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager mana
private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager,
boolean ownsManager, long appendDeadlineNanos) {
+ this(sfDir, segmentSizeBytes, manager, ownsManager,
+ SegmentManager.UNLIMITED_TOTAL_BYTES, appendDeadlineNanos);
+ }
+
+ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager,
+ boolean ownsManager, long maxTotalBytes, long appendDeadlineNanos) {
+ // Allocate the bound callback before constructing an owned manager.
+ // Field initializers have completed, but no engine-owned native/disk
+ // resource exists yet. If callback allocation throws, construction
+ // stops without a manager whose native path scratch could be orphaned.
+ this.deferredClose = createDeferredClose();
+ if (ownsManager && manager == null) {
+ manager = new SegmentManager(
+ segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes);
+ }
+
// sfDir == null → memory-only mode (non-SF async ingest). Same
// cursor architecture, no disk involvement; segments
// live in malloc'd native memory.
@@ -172,11 +261,9 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
// separate mkdir step needed here.
acquiredLock = SlotLock.acquire(sfDir);
} catch (Throwable t) {
- // The delegating constructors evaluate `new SegmentManager(...)`
- // BEFORE this body runs, so on a pre-try throw (e.g. slot lock
- // collision) an owned manager is already alive and would leak
- // its native path-scratch sink -- 256 bytes per failed
- // construction attempt. Close it before propagating.
+ // Callback creation and owned-manager construction have already
+ // completed. A slot-lock failure must close the owned manager's
+ // native path scratch before propagating.
if (ownsManager) {
try {
manager.close();
@@ -190,6 +277,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
this.sfDir = sfDir;
this.segmentSizeBytes = segmentSizeBytes;
this.manager = manager;
+ this.filesFacade = manager.filesFacade();
this.ownsManager = ownsManager;
this.appendDeadlineNanos = appendDeadlineNanos;
@@ -204,9 +292,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
// session before deciding to start fresh. Without this the engine
// would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs
// already on disk and corrupting ACK translation, trim, and replay.
- SegmentRing recovered = memoryMode ? null
- : SegmentRing.openExisting(sfDir, segmentSizeBytes);
- this.wasRecoveredFromDisk = recovered != null;
+ SegmentRing.Recovery recovery = memoryMode ? null
+ : SegmentRing.recover(filesFacade, sfDir, segmentSizeBytes);
+ SegmentRing recovered = recovery == null ? null : recovery.ring();
+ this.wasRecoveredFromDisk = recovery != null
+ && recovery.status() == SegmentRing.RecoveryStatus.RECOVERED;
if (recovered != null) {
ringInProgress = recovered;
// Seed ackedFsn to one below the lowest segment's baseSeq.
@@ -253,14 +343,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
// - trim ran before persist: segments are gone (so
// lowestBase is higher than watermark), watermark
// is stale; max picks lowestBase - 1.
- // open() returns null on any setup failure so a missing
- // mmap doesn't take down the engine -- we just fall
- // back to the bare lowestBase - 1 seed.
- watermarkInProgress = AckWatermark.open(sfDir);
+ // Segment-derived state cannot tell whether frames in the
+ // lowest surviving segment were acknowledged. Starting
+ // without the watermark would therefore expose acknowledged
+ // residue for replay, so disk recovery must fail closed when
+ // the file cannot be opened or mapped.
+ watermarkInProgress = AckWatermark.open(filesFacade, sfDir);
+ if (watermarkInProgress == null) {
+ throw new IllegalStateException(
+ "could not open required ack watermark for SF slot " + sfDir);
+ }
long baseSeed = lowestBase - 1;
- long watermarkFsn = watermarkInProgress != null
- ? watermarkInProgress.read()
- : AckWatermark.INVALID;
+ long watermarkFsn = watermarkInProgress.read();
// Reject watermarks past publishedFsn: a correctly
// operating prior session cannot have produced one, so
// a value above the on-disk frame ceiling is corruption
@@ -307,23 +401,58 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
// so the new session's first read() correctly reports
// INVALID (magic=0 on a freshly zero-filled file).
if (!memoryMode) {
- AckWatermark.removeOrphan(sfDir);
- watermarkInProgress = AckWatermark.open(sfDir);
+ AckWatermark.removeOrphan(filesFacade, sfDir);
+ watermarkInProgress = AckWatermark.open(filesFacade, sfDir);
+ if (watermarkInProgress == null) {
+ throw new IllegalStateException(
+ "could not open required ack watermark for SF slot " + sfDir);
+ }
}
MmapSegment initial;
String initialPath = null;
+ SfManifest initialManifest = null;
if (memoryMode) {
initial = MmapSegment.createInMemory(0L, segmentSizeBytes);
} else {
+ // Created WITHOUT the manifest-required flag: the flag is
+ // a durable promise that sf-manifest.bin exists, so it
+ // must only be stamped after the manifest itself is on
+ // disk. Stamping it at create time would leave a crash
+ // window (initial durable, manifest not yet created)
+ // whose recovery hard-fails with "segment requires
+ // missing manifest" even though nothing was lost.
initialPath = sfDir + "/sf-initial.sfa";
- initial = MmapSegment.create(initialPath, 0L, segmentSizeBytes);
+ initial = MmapSegment.create(filesFacade, initialPath, 0L, segmentSizeBytes);
}
try {
- ringInProgress = new SegmentRing(initial, segmentSizeBytes);
+ if (!memoryMode) {
+ // Ordering is load-bearing for crash recovery:
+ // 1. initial segment durable (header + dirent),
+ // 2. manifest created (its own create fsyncs),
+ // 3. flag stamped on the segment.
+ // A crash after (1) recovers via the legacy
+ // (manifest-less) path; after (2) via the manifest
+ // path with an unflagged-but-valid empty active;
+ // after (3) it is the steady state. Every window
+ // self-heals without operator action.
+ initial.syncHeader();
+ if (filesFacade.fsyncDir(sfDir) != 0) {
+ throw new MmapSegmentException(
+ "could not sync fresh SF segment directory " + sfDir);
+ }
+ initialManifest = SfManifest.create(filesFacade, sfDir, 0L, 0L);
+ initial.markManifestRequired();
+ }
+ ringInProgress = new SegmentRing(initial, segmentSizeBytes, initialManifest);
+ initialManifest = null;
} catch (Throwable t) {
initial.close();
+ if (initialManifest != null) {
+ initialManifest.close();
+ SfManifest.removeFile(filesFacade, sfDir);
+ }
if (initialPath != null) {
- Files.remove(initialPath);
+ filesFacade.remove(initialPath);
}
throw t;
}
@@ -480,7 +609,7 @@ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos
@Override
public synchronized void close() {
- if (closed) return;
+ if (closed && closeCompleted) return;
closed = true;
// Capture drain state BEFORE closing the ring — once the ring is
// closed, its accessors aren't safe to read. The active segment is
@@ -491,38 +620,212 @@ public synchronized void close() {
// against potentially-fresh server state — duplicate writes when
// the server has no dedup state for those messageSequences.
// Memory mode has no files to unlink.
- // The whole close sequence runs under try/finally so the slot lock
- // is ALWAYS released, even if manager/ring close or unlink throws —
- // otherwise a kernel-held flock outlives the engine and the next
- // sender for the same slot collides on a lock the dead engine
- // never released.
+ //
+ // Cleanup is gated on worker quiescence: releasing the ring,
+ // watermark, segment files or the slot lock while the manager worker
+ // is still mid service pass would let a replacement engine acquire
+ // the slot and have its files unlinked by the stale worker —
+ // store-and-forward data loss after restart. When the bounded worker
+ // join times out on an owned manager, cleanup OWNERSHIP TRANSFERS to
+ // the worker's exit path (deferUntilWorkerExit): the worker is
+ // provably the last thread able to touch the slot directory, so it
+ // releases everything the moment its in-flight pass finishes instead
+ // of leaking the slot until process exit.
+ //
+ // "Fully drained" includes BOTH the obvious case (every published
+ // FSN has been acked) AND the never-published case (publishedFsn
+ // < 0). The latter matters because a drainer adopting an empty
+ // orphan slot — segments filtered as empty by recovery, engine
+ // recreates a fresh sf-initial.sfa — would otherwise leave that
+ // fresh empty file behind, the next scanner finds it, adopts the
+ // slot again, and the cycle repeats forever (M6).
+ // Own try/catch so sabotaged/broken ring state cannot skip the
+ // quiescence barrier below or the slot-lock release in finishClose.
+ boolean drained = false;
try {
- // "Fully drained" includes BOTH the obvious case (every published
- // FSN has been acked) AND the never-published case (publishedFsn
- // < 0). The latter matters because a drainer adopting an empty
- // orphan slot — segments filtered as empty by recovery, engine
- // recreates a fresh sf-initial.sfa — would otherwise leave that
- // fresh empty file behind, the next scanner finds it, adopts the
- // slot again, and the cycle repeats forever (M6).
- boolean fullyDrained = sfDir != null
+ drained = sfDir != null
&& (ring.publishedFsn() < 0
|| ring.ackedFsn() >= ring.publishedFsn());
- // Each cleanup step in its own try/catch so a single failure
- // doesn't strand later cleanups — mirrors the constructor's
- // catch block. Without this, a throw from manager.deregister
- // or manager.close() would leave the ring mmap'd and any
- // residual .sfa files on disk, where the next sender can
- // adopt them and replay already-acked data.
+ } catch (Throwable ignored) {
+ }
+ final boolean fullyDrained = drained;
+ fullyDrainedForDeferredClose = fullyDrained;
+ // Each cleanup step in its own try/catch so a single failure
+ // doesn't strand later cleanups — mirrors the constructor's
+ // catch block. Without this, a throw from manager.deregister
+ // or manager.close() would leave the ring mmap'd and any
+ // residual .sfa files on disk, where the next sender can
+ // adopt them and replay already-acked data.
+ try {
+ manager.deregister(ring);
+ } catch (Throwable ignored) {
+ }
+ // Quiescence barrier. deregister alone only removes the entry
+ // from the registry — the worker may still be mid service pass
+ // for this ring (creating a spare file, trimming, unlinking).
+ boolean workerQuiescent = false;
+ if (ownsManager) {
+ // Stopping and reaping a private manager is a stronger barrier
+ // than waiting for this ring alone. Do it directly so a stuck
+ // worker consumes at most one workerJoinTimeoutMillis budget,
+ // rather than one here and a second one in manager.close().
try {
- manager.deregister(ring);
+ manager.close();
} catch (Throwable ignored) {
}
- if (ownsManager) {
- try {
- manager.close();
- } catch (Throwable ignored) {
+ try {
+ workerQuiescent = manager.isWorkerReaped();
+ } catch (Throwable ignored) {
+ }
+ } else {
+ // A shared manager must keep serving its other rings, so wait
+ // only for the deregistered ring's current pass to finish.
+ try {
+ workerQuiescent = manager.awaitRingQuiescence(ring);
+ } catch (Throwable ignored) {
+ }
+ }
+ if (!workerQuiescent && ownsManager) {
+ // Ownership handoff: manager.close() already stopped the worker
+ // loop (running=false), so the worker exits as soon as its
+ // in-flight pass finishes — it is merely slow, or wedged in a
+ // syscall. Either way it is the last thread able to touch the
+ // slot, so hand it the terminal cleanup instead of leaking the
+ // slot until process exit. completeDeferredClose and a retried
+ // close() race through the terminalCleanupClaimed CAS, so the
+ // cleanup runs exactly once and the worker never blocks on the
+ // engine monitor a retried close() holds while joining it.
+ boolean handedOff = false;
+ boolean registrationFailed = false;
+ try {
+ handedOff = manager.deferOwnedEngineCloseUntilWorkerExit(deferredClose);
+ } catch (Throwable ignored) {
+ // Unexpected monitor/VM failure carries no worker-liveness
+ // information. Ordinary handoff cannot allocate: both the
+ // callback and the manager's single slot were preallocated.
+ registrationFailed = true;
+ }
+ if (handedOff) {
+ LOG.error("SF manager worker did not quiesce during engine close; ring, watermark "
+ + "and slot-lock release are handed to the worker's exit path and run the "
+ + "moment its in-flight service pass finishes. The slot stays locked (and "
+ + "isCloseCompleted() false) until then, so no replacement engine can race "
+ + "the stale worker on slot {}", sfDir == null ? "" : sfDir);
+ return;
+ }
+ if (registrationFailed) {
+ // The handoff never registered and the worker was never
+ // observed — it must be presumed live and mid service pass.
+ // Retain every worker-reachable resource (ring, watermark,
+ // segment files, slot flock) and leave terminalCleanupClaimed
+ // unclaimed and closeCompleted false, exactly like the
+ // shared-manager leak branch: manager.close() above already
+ // stopped the worker loop, so a retried close() converges via
+ // isWorkerReaped() once the in-flight pass ends. The kernel
+ // releases the slot flock on process exit regardless.
+ LOG.error("SF worker-exit handoff registration failed during engine close; "
+ + "leaking the ring, watermark and slot lock so a possibly-live "
+ + "worker cannot corrupt a future engine on slot {}. close() may be "
+ + "invoked again to retry cleanup once the worker has exited.",
+ sfDir == null ? "" : sfDir);
+ return;
+ }
+ // Handoff rejected: the worker loop exited between the failed
+ // bounded join and the registration attempt (both sides share the
+ // manager's lock, so this observation is exact). A worker past
+ // its loop can never touch slot paths again — inline cleanup is
+ // as safe as a reaped worker.
+ workerQuiescent = true;
+ }
+ if (!workerQuiescent) {
+ // A shared manager keeps serving sibling rings, so it cannot use
+ // the whole-worker exit handoff above. Transfer cleanup to this
+ // ring's current pass instead. Registration and pass completion
+ // share the manager lock: true means the worker owns cleanup;
+ // false means the pass already finished and inline cleanup is safe.
+ boolean handedOff = false;
+ boolean registrationFailed = false;
+ try {
+ handedOff = manager.deferUntilRingQuiescent(ring, deferredClose);
+ } catch (Throwable ignored) {
+ registrationFailed = true;
+ }
+ if (handedOff) {
+ LOG.error("SF manager worker did not quiesce during engine close; ring, watermark "
+ + "and slot-lock release are handed to the current ring pass and run the "
+ + "moment that pass finishes. The slot stays locked (and "
+ + "isCloseCompleted() false) until then, so no replacement engine can "
+ + "race the stale worker on slot {}",
+ sfDir == null ? "" : sfDir);
+ return;
+ }
+ if (registrationFailed) {
+ LOG.error("SF ring-pass cleanup handoff registration failed during engine close; "
+ + "leaking the ring, watermark and slot lock so a possibly-live worker "
+ + "cannot corrupt a future engine on slot {}. The kernel releases the "
+ + "flock on process exit.", sfDir == null ? "" : sfDir);
+ return;
+ }
+ workerQuiescent = true;
+ }
+ if (!terminalCleanupClaimed.compareAndSet(false, true)) {
+ // A worker-exit handoff or earlier close owns the one-time
+ // ring/watermark cleanup. Once that work is published complete,
+ // this close may still retry an unconfirmed flock release.
+ retryFlockReleaseIfReady();
+ return;
+ }
+ finishClose(fullyDrained);
+ }
+
+ /**
+ * Terminal cleanup: closes the ring and watermark, unlinks drained
+ * segment files, releases the slot flock, and — only once the release
+ * is confirmed — latches {@link #closeCompleted}. Publish order
+ * is load-bearing: pools free the slot index the instant they observe
+ * {@code closeCompleted} (via {@code isSlotLockReleased()}), so it must
+ * never be visible while the flock is still held, or a replacement
+ * engine races the release and fails acquisition on a live slot.
+ * The caller must hold the engine monitor and
+ * must have established that the manager worker can no longer touch the
+ * slot directory (reaped, provably exited, or running this on its own
+ * exit path) AND have won the {@link #terminalCleanupClaimed} CAS — the
+ * claim token, not the engine monitor, is the exclusion between a
+ * worker-exit handoff and a retried close(), so ring/watermark/flock are
+ * never double-released and the worker's cleanup can never deadlock
+ * against a close() that holds the monitor while joining the worker.
+ */
+ private void finishClose(boolean fullyDrained) {
+ // On a fully-drained close, persist the final acked FSN through the
+ // still-mapped watermark BEFORE closing the ring/watermark and BEFORE
+ // unlinking any segment file. The manager persists the watermark only
+ // on its own tick, so it may lag the final ack. If any part of this
+ // covering barrier fails, retain the ring, watermark and slot flock:
+ // publishing the slot would let a successor recover acknowledged
+ // residue from a stale durable watermark. The shared retry driver owns
+ // convergence after the one-shot public close reports the failure.
+ if (fullyDrained && watermark != null) {
+ try {
+ long finalAckedFsn = ring.ackedFsn();
+ if (finalAckedFsn >= 0) {
+ watermark.write(finalAckedFsn);
+ watermark.sync();
+ if (filesFacade.fsyncDir(sfDir) != 0) {
+ throw new IllegalStateException(
+ "could not fsync SF slot directory before segment cleanup");
+ }
}
+ } catch (RuntimeException | Error e) {
+ terminalCleanupRetryReady = true;
+ terminalCleanupClaimed.set(false);
+ startFlockReleaseRetry();
+ throw e;
}
+ }
+ terminalCleanupRetryReady = false;
+
+ try {
+ RuntimeException durabilityFailure = null;
try {
ring.close();
} catch (Throwable ignored) {
@@ -541,24 +844,343 @@ public synchronized void close() {
} catch (Throwable ignored) {
}
}
- if (fullyDrained) {
+ if (fullyDrained && watermark != null) {
+ boolean segmentsRemoved = false;
try {
- unlinkAllSegmentFiles(sfDir);
+ segmentsRemoved = unlinkAllSegmentFiles(filesFacade, sfDir);
} catch (Throwable ignored) {
}
+ // Remove the watermark ONLY once every segment unlink is
+ // durable. Until the directory fsync succeeds, stable storage
+ // may still contain acknowledged segments and therefore still
+ // needs the durable watermark.
+ if (segmentsRemoved) {
+ if (filesFacade.fsyncDir(sfDir) != 0) {
+ durabilityFailure = new IllegalStateException(
+ "could not fsync SF slot directory after segment cleanup");
+ } else {
+ AckWatermark.removeOrphan(filesFacade, sfDir);
+ }
+ } else {
+ LOG.warn("close-time segment cleanup incomplete on slot {}; retaining the ack "
+ + "watermark so residual acknowledged segments stay covered -- the next "
+ + "engine on this slot recovers them as fully acked and retries the "
+ + "unlink on its own close", sfDir);
+ }
+ }
+ if (durabilityFailure != null) {
+ throw durabilityFailure;
+ }
+ } finally {
+ // The final watermark covering barrier has succeeded, so terminal
+ // resources can be released even if later best-effort cleanup or
+ // its post-unlink directory sync failed. The durable watermark
+ // still covers any segment a host crash restores.
+ //
+ // ORDER MATTERS: explicitly release the flock, verify it, and
+ // only then publish closeCompleted. Pools read isCloseCompleted()
+ // as "the slot dir is reusable" and free the slot index the moment
+ // it flips. SlotLock closes the fd once after confirmed unlock,
+ // but that close result cannot safely control publication because
+ // POSIX may consume the fd even when close reports failure.
+ Runnable hook = beforeFlockReleaseHook;
+ if (hook != null) {
try {
- AckWatermark.removeOrphan(sfDir);
+ hook.run();
} catch (Throwable ignored) {
+ // test-only; must never block the release
}
}
- } finally {
- if (slotLock != null) {
+ terminalResourcesCleaned = true;
+ retryFlockReleaseIfReady();
+ }
+ }
+
+ /**
+ * Installs a hook invoked after each failed shared-driver flock release.
+ * Test-only: makes persistent retry progress observable without sleeps.
+ */
+ @TestOnly
+ public static void setAfterFlockReleaseRetryFailureHook(Runnable hook) {
+ afterFlockReleaseRetryFailureHook = hook;
+ }
+
+ /**
+ * Installs a constructor fault hook immediately before the bound deferred
+ * close callback is created. Test-only: proves callback allocation failure
+ * occurs before an owned manager acquires native resources.
+ */
+ @TestOnly
+ public static void setBeforeDeferredCloseCreationHook(Runnable hook) {
+ beforeDeferredCloseCreationHook = hook;
+ }
+
+ /**
+ * Replaces the shared retry driver's inter-round park with a callback
+ * receiving the park duration the driver would have used. Test-only:
+ * makes the retry cadence inspectable and rounds coordinatable without
+ * wall-clock waits.
+ */
+ @TestOnly
+ public static void setFlockReleaseRetryParkOverride(LongConsumer override) {
+ flockReleaseRetryParkOverride = override;
+ }
+
+ /**
+ * Overrides creation of the single shared flock-release retry thread.
+ * Test-only: makes thread creation/start failure and persistent retry
+ * scalability deterministic without relying on scheduler timing.
+ */
+ @TestOnly
+ public static void setFlockReleaseRetryThreadFactory(ThreadFactory factory) {
+ synchronized (FLOCK_RELEASE_RETRY_LOCK) {
+ if (flockReleaseRetryThread != null || !FLOCK_RELEASE_RETRY_QUEUE.isEmpty()) {
+ throw new IllegalStateException("flock-release retry driver is active");
+ }
+ flockReleaseRetryThreadFactory = factory == null
+ ? DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY
+ : factory;
+ }
+ }
+
+ /**
+ * Installs a hook that {@link #finishClose} runs between the terminal
+ * cleanup and the slot-flock release. Test-only: makes the otherwise
+ * microsecond-wide cleanup/release window deterministic so tests can
+ * assert {@link #isCloseCompleted()} stays false until the release is
+ * confirmed.
+ */
+ @TestOnly
+ public void setBeforeFlockReleaseHook(Runnable hook) {
+ this.beforeFlockReleaseHook = hook;
+ }
+
+ /**
+ * Runs on a safe manager-worker handoff path when {@link #close()} moved
+ * cleanup ownership to either a shared manager's ring-pass completion or
+ * an owned manager's worker exit.
+ * Deliberately does NOT take the engine monitor: a retried close() can
+ * hold it while joining this very worker, and the thread cannot die until
+ * this method returns — the {@link #terminalCleanupClaimed} CAS provides
+ * the exactly-once exclusion instead, so the worker always exits promptly
+ * and the racing close() converges via {@code isWorkerReaped()}.
+ */
+ private void completeDeferredClose() {
+ if (!terminalCleanupClaimed.compareAndSet(false, true)) {
+ // A retried close() (or an earlier duplicate handoff) already ran
+ // the terminal cleanup.
+ return;
+ }
+ finishClose(fullyDrainedForDeferredClose);
+ LOG.info("deferred SF engine resource cleanup completed after manager-worker quiescence; "
+ + "slot release confirmed: {} [slot={}]",
+ closeCompleted, sfDir == null ? "" : sfDir);
+ }
+
+ private Runnable createDeferredClose() {
+ Runnable hook = beforeDeferredCloseCreationHook;
+ if (hook != null) {
+ hook.run();
+ }
+ return this::completeDeferredClose;
+ }
+
+ private boolean retryFlockReleaseIfReady() {
+ if (closeCompleted) {
+ return true;
+ }
+ if (!terminalResourcesCleaned) {
+ if (!terminalCleanupRetryReady
+ || !terminalCleanupClaimed.compareAndSet(false, true)) {
+ return false;
+ }
+ try {
+ finishClose(fullyDrainedForDeferredClose);
+ } catch (Throwable ignored) {
+ return false;
+ }
+ return closeCompleted;
+ }
+ boolean released;
+ if (slotLock != null) {
+ try {
+ released = slotLock.release();
+ } catch (Throwable ignored) {
+ released = false;
+ }
+ } else {
+ released = true;
+ }
+ if (released) {
+ closeCompleted = true;
+ Runnable listener = slotLockReleaseListener;
+ if (listener != null) {
try {
- slotLock.close();
+ listener.run();
} catch (Throwable ignored) {
- // best-effort; flock is also released by kernel on process exit
+ // A notification failure must not invalidate a confirmed release.
+ }
+ }
+ return true;
+ }
+ startFlockReleaseRetry();
+ return false;
+ }
+
+ private static void runFlockReleaseRetryDriver() {
+ // Capped exponential backoff: a persistent unlock failure must not
+ // burn a fixed 10 rounds of native release syscalls per second
+ // forever, but a transient failure must still recover promptly. The
+ // ramp doubles per fully-failed round from 100 ms up to 5 s.
+ long parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS;
+ while (true) {
+ final int roundSize;
+ synchronized (FLOCK_RELEASE_RETRY_LOCK) {
+ roundSize = FLOCK_RELEASE_RETRY_QUEUE.size();
+ if (roundSize == 0) {
+ flockReleaseRetryThread = null;
+ return;
+ }
+ }
+ boolean hasFailures = false;
+ boolean hasSuccesses = false;
+ for (int i = 0; i < roundSize; i++) {
+ final CursorSendEngine engine;
+ synchronized (FLOCK_RELEASE_RETRY_LOCK) {
+ engine = FLOCK_RELEASE_RETRY_QUEUE.pollFirst();
}
+ if (engine.retryFlockReleaseIfReady()) {
+ engine.flockReleaseRetryStarted.set(false);
+ hasSuccesses = true;
+ } else {
+ synchronized (FLOCK_RELEASE_RETRY_LOCK) {
+ FLOCK_RELEASE_RETRY_QUEUE.addLast(engine);
+ }
+ hasFailures = true;
+ Runnable hook = afterFlockReleaseRetryFailureHook;
+ if (hook != null) {
+ hook.run();
+ }
+ }
+ }
+ if (hasSuccesses) {
+ // Progress: the failure condition is clearing, so retry the
+ // remaining engines on the base cadence again.
+ parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS;
}
+ if (hasFailures) {
+ // Interruption must not abandon a retained flock, but clear
+ // the flag so subsequent parks still throttle retries.
+ Thread.interrupted();
+ LongConsumer parkOverride = flockReleaseRetryParkOverride;
+ if (parkOverride != null) {
+ parkOverride.accept(parkNanos);
+ } else {
+ LockSupport.parkNanos(parkNanos);
+ }
+ parkNanos = Math.min(parkNanos * 2, FLOCK_RELEASE_RETRY_MAX_PARK_NANOS);
+ }
+ }
+ }
+
+ private void startFlockReleaseRetry() {
+ if (!flockReleaseRetryStarted.compareAndSet(false, true)) {
+ return;
+ }
+ Throwable startFailure = null;
+ synchronized (FLOCK_RELEASE_RETRY_LOCK) {
+ FLOCK_RELEASE_RETRY_QUEUE.addLast(this);
+ if (flockReleaseRetryThread != null) {
+ // The driver may be parked on a ramped backoff; wake it so a
+ // freshly failed release gets its first driver retry promptly
+ // instead of inheriting older engines' full backoff.
+ LockSupport.unpark(flockReleaseRetryThread);
+ } else {
+ try {
+ Thread retryThread = flockReleaseRetryThreadFactory.newThread(
+ CursorSendEngine::runFlockReleaseRetryDriver);
+ if (retryThread == null) {
+ throw new IllegalStateException("retry thread factory returned null");
+ }
+ retryThread.setDaemon(true);
+ flockReleaseRetryThread = retryThread;
+ retryThread.start();
+ } catch (Throwable t) {
+ startFailure = t;
+ flockReleaseRetryThread = null;
+ CursorSendEngine queued;
+ while ((queued = FLOCK_RELEASE_RETRY_QUEUE.pollFirst()) != null) {
+ queued.flockReleaseRetryStarted.set(false);
+ }
+ }
+ }
+ }
+ if (startFailure == null) {
+ LOG.error("SF terminal cleanup or slot flock release failed during engine close; "
+ + "keeping closeCompleted=false and retrying on the shared driver so "
+ + "retired capacity recovers after the transient failure [slot={}]",
+ sfDir == null ? "" : sfDir);
+ } else {
+ // A later explicit close() or a pool's retired-slot probe
+ // (ensureFlockReleaseRetryScheduled) can still retry without
+ // repeating the one-time ring/watermark cleanup. The failed queue
+ // is cleared so the driver does not retain engines it cannot
+ // service.
+ LOG.error("Could not start SF flock-release retry driver; a retried close() "
+ + "or pool re-probe re-arms the retry [slot={}, error={}]",
+ sfDir == null ? "" : sfDir, String.valueOf(startFailure));
+ }
+ }
+
+ /**
+ * Whether {@link #close()} completed all cleanup, including a
+ * confirmed release of the SF slot lock — the flip is published
+ * strictly after explicit unlock succeeds, so observing {@code true}
+ * guarantees the slot dir is acquirable by a replacement engine. A false
+ * value after close means manager-worker quiescence could not be
+ * confirmed (or the flock release itself failed) and the
+ * worker-reachable resources were retained deliberately — either handed to the worker's exit path (owned manager),
+ * which flips this to true the moment the worker's in-flight pass
+ * finishes, or leaked until a retried close() (shared manager). Owners
+ * must not reuse the slot while this is false; pools may re-probe it to
+ * recover a retired slot's capacity once it flips.
+ *
+ * Deliberately unsynchronized ({@code closeCompleted} is volatile): pools
+ * probe this under their own capacity lock, and the deferred cleanup can
+ * hold the engine monitor through munmap/unlink syscalls — a synchronized
+ * getter would stall the pool's hot borrow path behind them.
+ */
+ public boolean isCloseCompleted() {
+ return closeCompleted;
+ }
+
+ /**
+ * Registers a callback for confirmed SF slot-lock release. If release
+ * already completed, invokes the callback before returning.
+ */
+ public void setSlotLockReleaseListener(Runnable listener) {
+ slotLockReleaseListener = listener;
+ if (listener != null && closeCompleted) {
+ listener.run();
+ }
+ }
+
+ /**
+ * Re-arms the shared terminal retry for an engine whose final watermark
+ * barrier or confirmed flock release is still pending and no longer
+ * scheduled because the retry driver thread failed to start (e.g. OOM at
+ * thread creation). A close still waiting for worker quiescence cannot be
+ * retried here because terminal cleanup may not race that worker.
+ * {@code Sender.close()} is one-shot by contract, so pool probes
+ * ({@code QwpWebSocketSender.isSlotLockReleased()}) call this to keep a
+ * retired slot's capacity recoverable instead of lost until process
+ * exit. Cheap unless the engine is in that orphan state (volatile reads,
+ * then one failed CAS when the retry is already scheduled), so probes
+ * may call it under their capacity lock.
+ */
+ public void ensureFlockReleaseRetryScheduled() {
+ if (!closeCompleted && (terminalResourcesCleaned || terminalCleanupRetryReady)) {
+ startFlockReleaseRetry();
}
}
@@ -660,36 +1282,138 @@ public long recoveredOrphanTipFsn() {
}
/**
- * Unlinks every {@code .sfa} file under {@code dir}. Called only on
- * clean shutdown when the ring confirms every published FSN has been
- * acked — at that moment the slot has no recoverable work and the
- * files are pure noise that would mislead the next sender's recovery.
- * Best-effort: logs and continues on failures, since we're already on
- * the close path.
+ * Ascending removal rank of a segment file name for
+ * {@link #unlinkAllSegmentFiles(String)}. {@code sf-initial.sfa} is
+ * always the fresh-start segment at baseSeq 0, so it ranks first.
+ * Spare files carry a monotonic generation ({@code sf-.sfa})
+ * assigned in creation == rotation == baseSeq order, so the parsed
+ * generation ranks them. Anything else with the extension is not a
+ * live segment and ranks last.
*/
- private static void unlinkAllSegmentFiles(String dir) {
- if (!io.questdb.client.std.Files.exists(dir)) return;
- long find = io.questdb.client.std.Files.findFirst(dir);
+ private static long segmentCleanupRank(String name) {
+ if ("sf-initial.sfa".equals(name)) {
+ return Long.MIN_VALUE;
+ }
+ if (name.length() == 23 && name.startsWith("sf-")) {
+ try {
+ return Long.parseUnsignedLong(name.substring(3, 19), 16);
+ } catch (NumberFormatException ignored) {
+ // fall through to the unrecognized-name rank
+ }
+ }
+ return Long.MAX_VALUE;
+ }
+
+ /**
+ * Unlinks every {@code .sfa} file under {@code dir}, then the SF
+ * manifest. Called only on clean shutdown when the ring confirms every
+ * published FSN has been acked — at that moment the slot has no
+ * recoverable work and the files are pure noise that would mislead the
+ * next sender's recovery.
+ *
+ * Crash safety comes from a single durable manifest update committed
+ * BEFORE the first unlink: collapsing the boundaries to
+ * {@code head == active} declares every frame below the active base
+ * trimmed, so any subset of surviving files recovers cleanly (stale
+ * files are discarded, a surviving active recovers as the whole chain,
+ * zero files with a leftover manifest is the recognized drain window
+ * and recovers as EMPTY). Removal still runs in ascending segment order
+ * and STOPS at the first failure; the retained ack watermark (== the
+ * final acked FSN) covers every surviving frame, so a successor replays
+ * nothing. Legacy slots without a manifest keep the pre-manifest
+ * argument: ascending-order removal preserves a contiguous top slice
+ * that passes FSN-contiguity.
+ *
+ * @return {@code true} only when enumeration succeeded and every
+ * {@code .sfa} file was confirmed removed — the caller keeps the ack
+ * watermark on {@code false} so residual acknowledged segments stay
+ * covered.
+ */
+ private static boolean unlinkAllSegmentFiles(FilesFacade filesFacade, String dir) {
+ if (!filesFacade.exists(dir)) return true;
+ long find = filesFacade.findFirst(dir);
if (find < 0) {
LOG.warn("close-time unlink could not enumerate {}; "
+ "any residual sf-*.sfa files will be picked up by the next recovery", dir);
- return;
+ return false;
}
- if (find == 0) return;
+ if (find == 0) return true;
+ ArrayList names = new ArrayList<>();
+ int rc = 1;
try {
- int rc = 1;
while (rc > 0) {
String name = io.questdb.client.std.Files.utf8ToString(
- io.questdb.client.std.Files.findName(find));
- rc = io.questdb.client.std.Files.findNext(find);
- if (name == null || !name.endsWith(".sfa")) continue;
- String path = dir + "/" + name;
- if (!io.questdb.client.std.Files.remove(path)) {
- LOG.warn("Failed to unlink fully-acked segment {} on close", path);
+ filesFacade.findName(find));
+ rc = filesFacade.findNext(find);
+ if (name != null && name.endsWith(".sfa")) {
+ names.add(name);
+ }
+ }
+ } finally {
+ filesFacade.findClose(find);
+ }
+ if (rc < 0) {
+ // A partial listing must not drive any unlink: removing only the
+ // files we happened to see could delete the segment holding the
+ // highest frame while a lower one survives, leaving residual
+ // state the retained watermark can no longer vouch for.
+ LOG.warn("close-time unlink could not fully enumerate {}; "
+ + "leaving all segment files for the next recovery", dir);
+ return false;
+ }
+ names.sort((a, b) -> {
+ int byRank = Long.compare(segmentCleanupRank(a), segmentCleanupRank(b));
+ return byRank != 0 ? byRank : a.compareTo(b);
+ });
+ // Manifest-aware drain protocol. Before removing anything, durably
+ // collapse the committed boundaries to head == active. From that
+ // moment EVERY crash state is a valid recovery input:
+ // - surviving data below the active base is "stale below head"
+ // (acked by definition here) and is discarded by recovery;
+ // - a surviving active-base segment recovers as the whole chain;
+ // - empty spares are validated extras;
+ // - zero segment files with the manifest still present is the
+ // recognized drain window and recovers as EMPTY.
+ // Without this barrier, deleting the file the manifest calls "head"
+ // would make the next startup fail on a slot that lost nothing.
+ // The manifest itself is removed last, after every segment unlink.
+ SfManifest manifest = null;
+ try {
+ try {
+ manifest = SfManifest.open(filesFacade, dir);
+ } catch (Throwable t) {
+ LOG.warn("close-time unlink could not read the SF manifest in {}; leaving "
+ + "all files for the next recovery", dir, t);
+ return false;
+ }
+ if (manifest != null && !names.isEmpty()) {
+ try {
+ long activeBase = manifest.activeBase();
+ manifest.update(activeBase, activeBase);
+ } catch (Throwable t) {
+ LOG.warn("close-time unlink could not commit drain boundaries in {}; "
+ + "leaving all files for the next recovery", dir, t);
+ return false;
+ }
+ }
+ for (int i = 0, n = names.size(); i < n; i++) {
+ String path = dir + "/" + names.get(i);
+ if (!filesFacade.remove(path)) {
+ LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the "
+ + "residual files stay a watermark-covered, manifest-consistent range", path);
+ return false;
}
}
} finally {
- io.questdb.client.std.Files.findClose(find);
+ if (manifest != null) {
+ manifest.close();
+ }
+ }
+ if (!SfManifest.removeFile(filesFacade, dir)) {
+ // Safe to proceed: a manifest with zero segment files is the
+ // recognized drain window and recovers as EMPTY.
+ LOG.warn("could not remove SF manifest in {} after full segment cleanup", dir);
}
+ return true;
}
}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 13a69f77..e83d907e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -49,6 +49,7 @@
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.LockSupport;
@@ -88,6 +89,29 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
* sub-second confirmation latency once the upload completes
* server-side. {@code 0} or negative disables the keepalive entirely.
*/
+ /**
+ * Bounded-await backstop for {@link #close()}: the maximum time close()
+ * waits for the I/O thread to stop (count down {@code shutdownLatch})
+ * before it loud-fails and delegates final teardown to the I/O thread's
+ * exit path. In the common case the round-2 in-flight connect cancellation
+ * (see {@link ConnectCancellation}) makes the I/O thread unwind and count
+ * the latch down within milliseconds, so this budget is never reached. It
+ * only fires in the pathological, astronomically-rare TOCTOU window where
+ * {@code cancel()}'s {@code closeTraffic()} lands between the pre-connect
+ * guard and native fd creation, so it is a no-op and the connect blocks the
+ * full OS SYN-retry (~60-130s per endpoint, possibly across several).
+ *
+ * {@code 30_000} ms is comfortably UNDER the sidecar's 120 s shutdown
+ * deadline (~4x headroom) yet ~1000x a healthy close's millisecond latch
+ * countdown, so it never prematurely abandons a legitimately draining
+ * close (the I/O thread only has to finish its current — now traffic-broken
+ * — native send/receive and run {@code ioLoop}'s finally). This bounds the
+ * WAIT inside close(), NOT the connect itself (rejected Option A): a
+ * legitimately slow SUCCESSFUL connect is unaffected — it is not concurrent
+ * with a close, and when close() does fire it cancels the in-flight connect
+ * rather than waiting it out.
+ */
+ public static final long DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS = 30_000L;
public static final long DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS = 200L;
public static final long DEFAULT_PARK_NANOS = 50_000L; // 50us idle backoff
/**
@@ -163,6 +187,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
// the durable watermark can lag behind the OK watermark.
private final ArrayDeque pendingDurable = new ArrayDeque<>();
private final ArrayDeque pendingDurablePool = new ArrayDeque<>();
+ // Race-safe cancellation handle for an in-flight connect attempt. Passed
+ // into reconnectFactory.reconnect(...) on the I/O thread so close() can
+ // break a connect blocked mid-attempt (a black-holed native connect that
+ // neither unpark nor interrupt cancels). See ConnectCancellation.
+ private final ConnectCancellation connectCancellation = new ConnectCancellation();
// Optional reconnect plumbing. When non-null, a wire failure triggers a
// reconnect attempt instead of a terminal fail(). The factory produces a
// fresh, connected+upgraded WebSocketClient.
@@ -200,7 +229,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
// by category. Includes both retriable and terminal outcomes — i.e. every
// server-side rejection observed regardless of how the loop reacted.
private final AtomicLong totalServerErrors = new AtomicLong();
- private WebSocketClient client;
+ private volatile WebSocketClient client;
// Optional: when non-null, every server-rejection error (retriable and
// terminal alike) is offered to the dispatcher for async delivery to the user's
// handler. Null disables async delivery entirely; the producer-side
@@ -225,6 +254,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
// it is engine.ackedFsn() + 1, so the first replayed frame on the new
// connection is wireSeq=0 and server-side cumulative ACKs still line up.
private long fsnAtZero;
+ // Bounded-await backstop budget for close() (see
+ // DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS). Overridable via
+ // setShutdownAwaitTimeoutMillis so tests can exercise the timeout branch
+ // deterministically without a multi-second real wait; production always
+ // uses the default. Read only on the owner thread inside close().
+ private long shutdownAwaitTimeoutMillis = DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS;
// Sticky flag: false until the very first time a live client is installed
// (either via the constructor in SYNC/OFF mode or via swapClient on a
// successful connect attempt in any mode). Once true, stays true. Used to
@@ -244,11 +279,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
// terminalError: the only writer runs on the I/O thread under the same
// first-writer-wins latch.
private volatile QwpDurableAckMismatchException capabilityGapTerminal;
- // Failed-stop hand-off flag: set by delegateEngineClose() when an owner's
- // close() could not stop the I/O thread and the engine close is therefore
- // performed by the I/O thread's exit path. Write-once, owner thread only;
- // read by the I/O thread strictly after its shutdown-latch countdown (see
- // the handshake contract on delegateEngineClose).
+ // Failed-stop hand-off callback: set when an owner could not stop the I/O
+ // thread and must defer worker-reachable cleanup until the thread exits.
+ // Read strictly after shutdownLatch.countDown(); see delegateClose().
+ private volatile Runnable delegatedClose;
+ // Engine-only hand-off retained for BackgroundDrainer, whose remaining
+ // resources are owned by its run method rather than by a sender.
private volatile boolean engineCloseDelegated;
// The latched terminal failure — THE exception every checkError() call
// rethrows. Write-once for the loop's lifetime: the only writer is
@@ -268,11 +304,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
private long nextWireSeq;
private volatile SenderProgressDispatcher progressDispatcher;
// Frames sent during the post-reconnect catch-up window — i.e. frames
- // whose FSN was already published before the wire dropped. A non-zero
+ // whose FSN completed a send on an earlier connection. A non-zero
// value confirms replay is working; a sustained nonzero rate means
// the connection is flapping and replay is doing real work each cycle.
- // Set at swapClient time to publishedFsn at that moment; cleared back
- // to -1 once trySendOne has caught up past it. Used to count replay
+ // Snapshotted when an established connection enters reconnect; cleared
+ // back to -1 once trySendOne has caught up past it. Used to count replay
// frames without a per-frame branch on the steady-state path.
private long replayTargetFsn = -1L;
// Recovered orphaned deferred tail: frames [orphanSkipStartFsn ..
@@ -763,8 +799,74 @@ public synchronized void close() {
// would block forever. isAlive()==false also covers the normal
// post-exit case where the latch is already counted down.
if (t.isAlive()) {
+ // Break a native send/receive before joining. Full client close
+ // must remain after the worker exit because it frees buffers the
+ // worker may still access.
+ WebSocketClient activeClient = client;
+ if (activeClient != null) {
+ try {
+ activeClient.closeTraffic();
+ } catch (Throwable e) {
+ // A custom transport that cannot safely cancel traffic
+ // must not be destructively closed while the worker is
+ // live. Fail without joining indefinitely; the worker's
+ // exit path retains ownership of final cleanup.
+ throw new LineSenderException(
+ "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; "
+ + "client/engine teardown is delegated to the I/O thread's exit path",
+ e
+ );
+ }
+ }
+ // Cancel an in-flight connect attempt not yet installed as the
+ // `client` field: async initial connect leaves `client` null
+ // and a mid-flight reconnect leaves it pointing at the stale
+ // pre-drop client, so neither is reachable by the closeTraffic()
+ // above. The reconnect factory publishes the connecting client
+ // here before it blocks, so breaking its traffic unwinds a
+ // black-holed native connect (connect_timeout=0 => OS SYN-retry,
+ // ~60-130s) that would otherwise pin the untimed await below.
+ // Same loud-fail contract as the field client: a transport that
+ // cannot safely shut down traffic is not destructively closed;
+ // the worker's exit path retains ownership of final cleanup.
try {
- shutdownLatch.await();
+ connectCancellation.cancel();
+ } catch (Throwable e) {
+ throw new LineSenderException(
+ "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; "
+ + "client/engine teardown is delegated to the I/O thread's exit path",
+ e
+ );
+ }
+ try {
+ // Bounded backstop: never wait forever. The round-2 in-flight
+ // connect cancellation makes the I/O thread unwind and count
+ // the latch down in milliseconds, so this await returns true
+ // almost always. The timeout branch fires only in the
+ // pathological uninterruptible-connect TOCTOU window where
+ // cancel()'s closeTraffic() was a no-op and the connect blocks
+ // the full OS SYN-retry -- guaranteeing close() still returns
+ // (bounded, comfortably under the sidecar's 120s deadline)
+ // rather than hanging on an untimed await.
+ if (!shutdownLatch.await(shutdownAwaitTimeoutMillis, TimeUnit.MILLISECONDS)) {
+ // Latch still up after the budget: the I/O thread did not
+ // stop. Same failed-stop protocol as the interrupt branch
+ // below -- loud-fail without touching the client/engine
+ // (freeing native buffers under a still-live I/O thread
+ // risks a C5 SEGV; a quiet return would let the owner
+ // unmap the engine under it). The I/O thread's own exit
+ // path (ioLoop's finally) retains ownership of final
+ // cleanup; QwpWebSocketSender.close() keys its
+ // ioThreadStopped guard on this throw and BackgroundDrainer
+ // switches to delegateEngineClose(). ioThread stays set
+ // (not nulled below, since we throw) so a duplicate close()
+ // re-signals rather than silently succeeding.
+ throw new LineSenderException(
+ "cursor I/O thread did not stop: close() timed out after "
+ + shutdownAwaitTimeoutMillis + "ms awaiting shutdown; "
+ + "client/engine teardown is delegated to the I/O "
+ + "thread's exit path");
+ }
} catch (InterruptedException e) {
// Re-assert the flag for the caller's stack, then decide.
// If the I/O thread has genuinely not exited (latch still
@@ -815,27 +917,20 @@ public synchronized void close() {
}
/**
- * Failed-stop hand-off for the engine. Called by an owner whose
- * {@link #close()} threw because the I/O thread would not stop: the owner
- * must not free the engine (munmap/Unsafe.free of segment memory) while
- * the thread may still touch it with raw {@code Unsafe} reads. Setting
- * the delegation flag makes the I/O thread run {@code engine.close()} on
- * its exit path, strictly after its last engine access and after the
- * shutdown-latch countdown — releasing the slot lock as soon as the
- * stuck wire call resolves (bounded by OS timeouts) instead of leaking
- * the mapping and lock forever.
- *
- * Returns {@code true} when the I/O thread is still live and has adopted
- * the engine close; {@code false} when the thread has already exited —
- * the caller must close the engine itself.
- *
- * Memory model — the classic store/load handshake: this method writes the
- * volatile flag, then reads the latch count; the exit path counts the
- * latch down, then reads the flag. Under the sequential consistency of
- * volatile (and AQS latch state) accesses, if this method observes the
- * latch still up, the exit path is guaranteed to observe the flag — no
- * missed close. If both sides act, {@link CursorSendEngine#close()} is
- * synchronized and idempotent, so the double close is benign.
+ * Hands complete owner cleanup to the I/O thread when it could not be
+ * stopped. The callback runs after the thread's last client, buffer, and
+ * engine access. Returns false if the thread already exited, in which case
+ * the caller must run the callback. The callback itself must be idempotent:
+ * the latch/callback handshake guarantees execution but permits both sides
+ * to race after the countdown.
+ */
+ public boolean delegateClose(Runnable closeCallback) {
+ delegatedClose = closeCallback;
+ return shutdownLatch.getCount() != 0L;
+ }
+
+ /**
+ * Engine-only failed-stop hand-off used by BackgroundDrainer.
*/
public boolean delegateEngineClose() {
engineCloseDelegated = true;
@@ -983,6 +1078,17 @@ public void setProgressDispatcher(SenderProgressDispatcher dispatcher) {
this.progressDispatcher = dispatcher;
}
+ /**
+ * Test seam: shrink the {@link #close()} bounded-await backstop
+ * (default {@link #DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS}) so a test can
+ * exercise the timeout branch deterministically without a multi-second
+ * real wait. Production never calls this. Set before {@link #close()}.
+ */
+ @TestOnly
+ public void setShutdownAwaitTimeoutMillis(long millis) {
+ this.shutdownAwaitTimeoutMillis = millis;
+ }
+
public synchronized void start() {
if (ioThread != null) {
throw new IllegalStateException("already started");
@@ -1114,10 +1220,14 @@ private void clearDurableAckTracking() {
* before the first connect attempt — see {@link #failPaced}.
*/
private void connectLoop(Throwable initial, String phase, long paceFirstAttemptMillis) {
- if (reconnectFactory == null || !running) {
+ if (!running) {
+ return;
+ }
+ if (reconnectFactory == null) {
recordFatal(initial);
return;
}
+ snapshotReplayTarget();
LOG.warn("cursor I/O loop entering {} loop: {}",
phase, initial.getMessage());
long outageStartNanos = System.nanoTime();
@@ -1156,7 +1266,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
attempts++;
totalReconnectAttempts.incrementAndGet();
try {
- WebSocketClient newClient = reconnectFactory.reconnect();
+ WebSocketClient newClient = reconnectFactory.reconnect(connectCancellation);
if (newClient != null) {
if (!running) {
// close() ran while this connect attempt was in
@@ -1670,15 +1780,14 @@ private void ioLoop() {
}
}
shutdownLatch.countDown();
- // Failed-stop hand-off (see delegateEngineClose): the owner could
- // not free the engine safely while this thread was alive, so the
- // engine close — and with it the slot-lock release — happens
- // here, strictly after this thread's last engine access. The flag
- // is read only after the countDown: the store/load pairing with
- // delegateEngineClose's flag-write-then-latch-read guarantees
- // either this branch or the owner's fallback runs (or both —
- // engine.close() is idempotent).
- if (engineCloseDelegated) {
+ Runnable closeCallback = delegatedClose;
+ if (closeCallback != null) {
+ try {
+ closeCallback.run();
+ } catch (Throwable ignored) {
+ // The owner callback logs individual cleanup failures.
+ }
+ } else if (engineCloseDelegated) {
try {
engine.close();
} catch (Throwable ignored) {
@@ -1823,6 +1932,23 @@ private void sendDurableAckKeepaliveIfDue() {
}
}
+ /**
+ * Freezes the highest FSN that completed a send on the connection which
+ * just failed. Frames published before the first connection, or while a
+ * reconnect is in progress, have not been sent and therefore do not belong
+ * to the replay metric. Keep an older, higher target when another failure
+ * interrupts catch-up so the remaining frames still count as re-sends on
+ * the following connection.
+ */
+ private void snapshotReplayTarget() {
+ if (hasEverConnected && nextWireSeq > 0L) {
+ long lastSentFsn = fsnAtZero + (nextWireSeq - 1L);
+ if (lastSentFsn > replayTargetFsn) {
+ replayTargetFsn = lastSentFsn;
+ }
+ }
+ }
+
/**
* Reset wire state for a fresh connection: install the new client,
* realign {@code fsnAtZero} to the next unacked FSN, restart wire
@@ -1851,11 +1977,12 @@ private void swapClient(WebSocketClient newClient) {
long replayStart = engine.ackedFsn() + 1L;
this.fsnAtZero = replayStart;
this.nextWireSeq = 0L;
- // Snapshot publishedFsn at swap time — frames at FSN ≤ this value
- // were already on the wire before the drop and will be replayed.
- // trySendOne resets replayTargetFsn to -1 once we cross the boundary.
- long pubAtSwap = engine.publishedFsn();
- this.replayTargetFsn = pubAtSwap >= replayStart ? pubAtSwap : -1L;
+ // snapshotReplayTarget froze the completed-send boundary when the
+ // outage began. ACK progress can move replayStart past that boundary;
+ // in that case no frame needs re-sending on this connection.
+ if (replayTargetFsn < replayStart) {
+ replayTargetFsn = -1L;
+ }
// Drop any durable-ack tracking from the previous connection. The
// new connection will re-OK every replayed batch and the server
// re-emits cumulative durable-ack watermarks from scratch, so
@@ -2034,6 +2161,104 @@ private boolean tryRetireOrphanTail() {
@FunctionalInterface
public interface ReconnectFactory {
WebSocketClient reconnect() throws Exception;
+
+ /**
+ * Cancellable variant of {@link #reconnect()}. The loop passes a
+ * per-attempt {@link ConnectCancellation} so a transport that blocks
+ * inside a native connect can publish the in-flight client to the
+ * handle BEFORE it blocks; {@link #close()} then breaks that client's
+ * traffic to unwind a black-holed connect promptly, rather than falling
+ * back to the bounded {@code shutdownLatch.await(...)} backstop (which
+ * only bounds the WAIT, not the connect).
+ *
+ * Default: ignore the handle and delegate to {@link #reconnect()} --
+ * a transport that cannot publish its in-flight client is simply not
+ * cancellable mid-connect, and {@code close()} falls back to its
+ * existing field-client {@code closeTraffic()} / loud-fail path. The
+ * default keeps this a {@link FunctionalInterface} and preserves
+ * source compatibility for existing lambda / method-reference
+ * implementors (e.g. {@link #connectWithRetry}).
+ */
+ default WebSocketClient reconnect(ConnectCancellation cancellation) throws Exception {
+ return reconnect();
+ }
+ }
+
+ /**
+ * Race-safe cancellation handle for a single in-flight connect attempt.
+ * Owned per-loop and passed into {@link ReconnectFactory#reconnect(ConnectCancellation)}
+ * on the I/O thread. The transport's connect walk {@link #publish}es the
+ * {@link WebSocketClient} it is about to block on before the blocking
+ * {@code connect()}, and {@link #clear}s it once the attempt is installed
+ * or disposed. {@link #close()} (owner thread) calls {@link #cancel()} to
+ * break the in-flight client's traffic path so a black-holed native
+ * connect unwinds and the I/O thread counts down the shutdown latch.
+ *
+ * Java 8: two volatiles, no locks. Visibility/ordering:
+ *
+ *
{@code inFlight} is written by the I/O thread (publish/clear) and
+ * read by the owner thread (cancel).
+ *
{@code cancelled} is written by the owner thread (cancel) and read
+ * by the I/O thread (the pre-connect guard).
+ *
+ * The publish (write inFlight, then read cancelled) / cancel (write
+ * cancelled, then read inFlight) handshake covers the close-vs-connect
+ * race in both directions: if the guard observes {@code cancelled==false}
+ * then, by the volatile total order, {@code cancel()} observed the
+ * published client and broke its traffic; if the guard observes
+ * {@code cancelled==true} it skips the blocking connect entirely.
+ */
+ public static final class ConnectCancellation {
+ // The client the connect walk is currently about to block / blocked
+ // on. Written by the I/O thread only (publish/clear); read by the
+ // owner thread (cancel). volatile for cross-thread visibility.
+ private volatile WebSocketClient inFlight;
+ // Latched once close() requested cancellation. Written by the owner
+ // thread (cancel); read by the I/O thread's pre-connect guard.
+ private volatile boolean cancelled;
+
+ public boolean isCancelled() {
+ return cancelled;
+ }
+
+ /**
+ * I/O-thread hook: record the client the walk is about to block on,
+ * BEFORE the blocking {@code connect()}. Pairs with {@link #cancel()}
+ * so an attempt that publishes after cancellation is caught by the
+ * caller's {@link #isCancelled()} guard, and an attempt already
+ * blocked in {@code connect()} is broken by {@code cancel()}.
+ */
+ public void publish(WebSocketClient client) {
+ inFlight = client;
+ }
+
+ /**
+ * I/O-thread hook: the walk is done with the in-flight attempt
+ * (installed on success, disposed on failure). Drop the reference so a
+ * later {@link #cancel()} cannot touch a client the walk no longer
+ * owns. Single writer (I/O thread) for publish/clear, so no CAS is
+ * needed.
+ */
+ public void clear() {
+ inFlight = null;
+ }
+
+ /**
+ * Owner-thread hook from {@link #close()}: request cancellation, then
+ * break the traffic path of any in-flight connect so the I/O thread's
+ * connect walk unwinds. {@code cancelled} is written before reading
+ * {@code inFlight} to pair with {@link #publish(WebSocketClient)}. May
+ * throw when the transport cannot safely shut down traffic --
+ * {@code close()} maps that to its existing loud-fail, exactly as it
+ * does for the field client's {@code closeTraffic()}.
+ */
+ void cancel() {
+ cancelled = true;
+ WebSocketClient c = inFlight;
+ if (c != null) {
+ c.closeTraffic();
+ }
+ }
}
/**
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
index 78e5db9f..ff7a00da 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
@@ -63,10 +63,12 @@ public final class MmapSegment implements QuietCloseable {
public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian
public static final int FRAME_HEADER_SIZE = 8; // u32 crc + u32 payloadLen
public static final int HEADER_SIZE = 24;
+ public static final byte MANIFEST_REQUIRED_FLAG = 1;
public static final byte VERSION = 1;
private static final int[] CRC32C_TABLE = buildCrc32cTable();
private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class);
+ private final FilesFacade filesFacade;
private final String path;
private final long sizeBytes;
// memoryBacked: true when the segment buffer lives in malloc'd native
@@ -95,6 +97,10 @@ public final class MmapSegment implements QuietCloseable {
// because the consumer must see writes in publication order — once the
// producer bumps publishedCursor, every byte before it is fully written.
private volatile long publishedCursor;
+ // Monotonic in-memory link to the segment that immediately follows this
+ // one. SegmentRing publishes it before promoting the successor to active;
+ // close deliberately retains it so a cursor can advance after head trim.
+ private volatile MmapSegment successor;
// Bytes between the last valid frame and the file end that look like an
// attempted-but-invalid frame write (non-zero bytes at the bail-out
// position). Zero for fresh segments and for cleanly partially-filled
@@ -102,9 +108,10 @@ public final class MmapSegment implements QuietCloseable {
// recovery callers for diagnostics. Final after construction.
private final long tornTailBytes;
- private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
+ private MmapSegment(FilesFacade filesFacade, String path, int fd, long mmapAddress, long sizeBytes,
long baseSeq, long initialCursor, long frameCount,
boolean memoryBacked, long tornTailBytes) {
+ this.filesFacade = filesFacade;
this.path = path;
this.fd = fd;
this.mmapAddress = mmapAddress;
@@ -145,9 +152,13 @@ public static MmapSegment create(String path, long baseSeq, long sizeBytes) {
* that filesystem only.
*/
public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes) {
+ return create(ff, path, baseSeq, sizeBytes, false);
+ }
+
+ static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes, boolean manifestRequired) {
long pathPtr = ff.allocNativePath(path);
try {
- return create(ff, pathPtr, path, baseSeq, sizeBytes);
+ return create(ff, pathPtr, path, baseSeq, sizeBytes, manifestRequired);
} finally {
ff.freeNativePath(pathPtr);
}
@@ -162,13 +173,17 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long
* per-call {@code byte[]} + native-malloc the way the String overload does.
*/
public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) {
+ return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, false);
+ }
+
+ static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes, boolean manifestRequired) {
if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) {
throw new IllegalArgumentException(
"sizeBytes too small for header + one minimal frame: " + sizeBytes);
}
- int fd = ff.openCleanRW(pathPtr);
+ int fd = ff.openRWExclusive(pathPtr);
if (fd < 0) {
- throw new MmapSegmentException("openCleanRW failed for " + displayPath);
+ throw new MmapSegmentException("exclusive create failed for " + displayPath);
}
// Reserve real disk blocks and advance EOF to sizeBytes in one
// call. ENOSPC surfaces here, before the producer thread starts
@@ -186,21 +201,21 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat
}
long addr = Files.FAILED_MMAP_ADDRESS;
try {
- addr = Files.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
+ addr = ff.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
if (addr == Files.FAILED_MMAP_ADDRESS) {
throw new MmapSegmentException("mmap failed for " + displayPath);
}
// Header goes straight into the mapping — no separate write syscall.
Unsafe.getUnsafe().putInt(addr, FILE_MAGIC);
Unsafe.getUnsafe().putByte(addr + 4, VERSION);
- Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags
+ Unsafe.getUnsafe().putByte(addr + 5, manifestRequired ? MANIFEST_REQUIRED_FLAG : (byte) 0); // flags
Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved
Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
- return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L);
+ return new MmapSegment(ff, displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L);
} catch (Throwable t) {
if (addr != Files.FAILED_MMAP_ADDRESS) {
- Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT);
+ ff.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT);
}
ff.close(fd);
// mmap (or header writes) failed after a successful allocate —
@@ -234,7 +249,7 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) {
Unsafe.getUnsafe().putShort(addr + 6, (short) 0);
Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
- return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L);
+ return new MmapSegment(null, null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L);
} catch (Throwable t) {
Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT);
throw t;
@@ -280,7 +295,11 @@ public static MmapSegment openExisting(String path) {
public static MmapSegment openExisting(FilesFacade ff, String path) {
long fileSize = ff.length(path);
if (fileSize < HEADER_SIZE) {
- throw new MmapSegmentException("file shorter than header: " + path + " size=" + fileSize);
+ // Corruption, not an operational error: the bytes themselves prove
+ // this cannot be a whole segment (a create() is never durable at a
+ // sub-header size — allocate() reserves the full extent up front).
+ throw new MmapSegmentCorruptionException(
+ "file shorter than header: " + path + " size=" + fileSize);
}
int fd = ff.openRW(path);
if (fd < 0) {
@@ -288,17 +307,22 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
}
long addr = Files.FAILED_MMAP_ADDRESS;
try {
- addr = Files.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
+ addr = ff.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
if (addr == Files.FAILED_MMAP_ADDRESS) {
throw new MmapSegmentException("mmap failed for " + path);
}
int magic = Unsafe.getUnsafe().getInt(addr);
if (magic != FILE_MAGIC) {
- throw new MmapSegmentException(
+ throw new MmapSegmentCorruptionException(
"bad magic in " + path + ": 0x" + Integer.toHexString(magic));
}
byte version = Unsafe.getUnsafe().getByte(addr + 4);
if (version != VERSION) {
+ // Deliberately NOT the corruption subtype: an unsupported
+ // version is a well-formed segment written by a different
+ // client build (e.g. after a downgrade). Quarantine-renaming
+ // it would strand its frames for the writer that CAN read it;
+ // failing recovery keeps the slot intact for that writer.
throw new MmapSegmentException("unsupported version in " + path + ": " + version);
}
long baseSeq = Unsafe.getUnsafe().getLong(addr + 8);
@@ -310,7 +334,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
// checks (which would place the segment last in baseSeq order
// and trip the FSN-gap throw, taking the whole recovery down).
if (baseSeq < 0L) {
- throw new MmapSegmentException(
+ throw new MmapSegmentCorruptionException(
"bad baseSeq in " + path + ": " + baseSeq);
}
long lastGood = scanFrames(addr, fileSize);
@@ -324,10 +348,10 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
+ "Investigate disk health or unexpected writer crash.",
path, tornTail, lastGood, fileSize, count);
}
- return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail);
+ return new MmapSegment(ff, path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail);
} catch (Throwable t) {
if (addr != Files.FAILED_MMAP_ADDRESS) {
- Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT);
+ ff.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT);
}
ff.close(fd);
// The header reads above (magic/version/baseSeq) run before
@@ -343,7 +367,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
// covers the header block and any future reader placed ahead of the
// scan.
if (isMmapAccessFault(t)) {
- throw new MmapSegmentException(
+ throw new MmapSegmentCorruptionException(
"unreadable mapped header page in " + path
+ " (unbacked/sparse page 0): " + t.getMessage(), t);
}
@@ -375,16 +399,37 @@ public void close() {
if (memoryBacked) {
Unsafe.free(mmapAddress, sizeBytes, MemoryTag.NATIVE_DEFAULT);
} else {
- Files.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT);
+ filesFacade.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT);
}
mmapAddress = 0;
}
if (fd >= 0) {
- Files.close(fd);
+ filesFacade.close(fd);
fd = -1;
}
}
+ boolean manifestRequired() {
+ return !memoryBacked && (Unsafe.getUnsafe().getByte(mmapAddress + 5) & MANIFEST_REQUIRED_FLAG) != 0;
+ }
+
+ void markManifestRequired() {
+ if (memoryBacked || manifestRequired()) {
+ return;
+ }
+ Unsafe.getUnsafe().putByte(mmapAddress + 5, MANIFEST_REQUIRED_FLAG);
+ syncHeader();
+ }
+
+ void syncHeader() {
+ if (memoryBacked) {
+ return;
+ }
+ if (filesFacade.msync(mmapAddress, HEADER_SIZE, false) != 0 || filesFacade.fsync(fd) != 0) {
+ throw new MmapSegmentException("could not sync segment header " + path);
+ }
+ }
+
public boolean isFull() {
return capacityRemaining() <= 0;
}
@@ -398,7 +443,7 @@ public void msync() {
if (memoryBacked) return; // no on-disk pages to flush
long pub = publishedCursor;
if (pub > HEADER_SIZE) {
- Files.msync(mmapAddress, pub, false);
+ filesFacade.msync(mmapAddress, pub, false);
}
}
@@ -438,6 +483,21 @@ public long sizeBytes() {
return sizeBytes;
}
+ void linkSuccessor(MmapSegment next) {
+ if (next == null) {
+ throw new IllegalArgumentException("successor must not be null");
+ }
+ MmapSegment existing = successor;
+ if (existing != null && existing != next) {
+ throw new IllegalStateException("segment successor already linked");
+ }
+ successor = next;
+ }
+
+ MmapSegment successor() {
+ return successor;
+ }
+
/**
* Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]}
* starting at the current append cursor, then advances both cursors
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java
new file mode 100644
index 00000000..34a89f38
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java
@@ -0,0 +1,50 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+/**
+ * Positively-identified segment corruption: the file's own bytes prove it is
+ * not (or no longer) a readable SF segment — truncated below the fixed header,
+ * wrong magic, a negative {@code baseSeq}, or an unreadable (unbacked/torn)
+ * header page. Distinct from its parent {@link MmapSegmentException}, which
+ * recovery treats as an operational failure (open/mmap error on a file
+ * whose contents may be perfectly intact) and must therefore be fatal.
+ *
+ * Recovery quarantines corruption (rename to {@code .corrupt}) and
+ * relies on manifest boundaries / FSN contiguity to decide whether the
+ * quarantined file was load-bearing; operational failures always abort
+ * startup so a transient {@code EMFILE}/{@code ENOMEM} can never silently
+ * drop durable frames.
+ */
+public final class MmapSegmentCorruptionException extends MmapSegmentException {
+
+ public MmapSegmentCorruptionException(String message) {
+ super(message);
+ }
+
+ public MmapSegmentCorruptionException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java
index eec0c0d9..e54eecb5 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java
@@ -29,8 +29,14 @@
* too short for header, etc. Indicates the segment is unusable, not that
* the disk is full (the latter surfaces as backpressure on the producer
* via {@link io.questdb.client.cutlass.qwp.client.LineSenderException}).
+ *
+ * Recovery distinguishes two flavors: this base type marks operational
+ * failures (open/mmap/enumeration errors — the file's contents may be fine,
+ * so recovery must fail closed), while the {@link MmapSegmentCorruptionException}
+ * subtype marks positively-identified corruption in the file's own
+ * bytes, which recovery may quarantine instead of aborting.
*/
-public final class MmapSegmentException extends RuntimeException {
+public class MmapSegmentException extends RuntimeException {
public MmapSegmentException(String message) {
super(message);
}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java
index 3b7cff60..5319ee65 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java
@@ -34,8 +34,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.concurrent.locks.LockSupport;
+import java.util.function.LongSupplier;
/**
* Background worker that keeps every registered {@link SegmentRing} supplied
@@ -49,27 +52,32 @@
* in a JVM). Polls each ring on a configurable tick (default 1 ms) — short
* enough that a producer rarely sees {@link SegmentRing#BACKPRESSURE_NO_SPARE}
* in the steady state, long enough that an idle JVM doesn't burn CPU.
- *
- * baseSeq race window: the spare is created with
- * {@code baseSeq = ring.nextSeqHint()} as observed by the manager. If the
- * producer thread appends more frames before the rotation actually fires,
- * the spare's baseSeq will be stale and {@link SegmentRing#appendOrFsn} will
- * throw on the mismatch check. In practice this is benign — by the time
- * {@link SegmentRing#needsHotSpare()} returns true the producer has very
- * little room left in the active segment, and the manager polls fast enough
- * to install before the producer fills the rest. Hardening to make the race
- * impossible (lazy header write at rotation time) is a separate refinement
- * deferred to PR2.
*/
public final class SegmentManager implements QuietCloseable {
public static final long DEFAULT_POLL_NANOS = 1_000_000L; // 1 ms
public static final long DISK_FULL_LOG_THROTTLE_NANOS = 30_000_000_000L; // 30 s
public static final long UNLIMITED_TOTAL_BYTES = Long.MAX_VALUE;
+ private static final int ENTRY_DEREGISTERED = 3;
+ private static final int ENTRY_DEREGISTERED_IN_SERVICE = 2;
+ private static final int ENTRY_IN_SERVICE = 1;
+ private static final int ENTRY_REGISTERED = 0;
+ private static final AtomicReferenceFieldUpdater ENTRY_CLEANUP_UPDATER =
+ AtomicReferenceFieldUpdater.newUpdater(RingEntry.class, Runnable.class, "quiescenceCleanup");
+ private static final AtomicIntegerFieldUpdater ENTRY_STATE_UPDATER =
+ AtomicIntegerFieldUpdater.newUpdater(RingEntry.class, "state");
private static final Logger LOG = LoggerFactory.getLogger(SegmentManager.class);
+ private static final int MAX_TRIMS_PER_RING_PASS = 64;
+ private static final long TRIM_RETRY_INITIAL_NANOS = 4_000_000L;
+ private static final long TRIM_RETRY_MAX_NANOS = 1_024_000_000L;
+ private static final int TRIM_RETRY_NONE = 0;
+ private static final int TRIM_RETRY_POST_BARRIER = 3;
+ private static final int TRIM_RETRY_PRE_BARRIER = 1;
+ private static final int TRIM_RETRY_UNLINK = 2;
private static final long WORKER_JOIN_TIMEOUT_MILLIS = 5_000L;
private final AtomicLong fileGeneration = new AtomicLong();
+ private final FilesFacade filesFacade;
private final Object lock = new Object();
private final long maxTotalBytes;
// Reused by the manager worker thread to build spare-segment paths
@@ -86,6 +94,22 @@ public final class SegmentManager implements QuietCloseable {
private final ObjList ringSnapshot = new ObjList<>();
private final ObjList rings = new ObjList<>();
private final long segmentSizeBytes;
+ private final LongSupplier ticks;
+ // Reused by the worker for one bounded trim quantum. Keeping the unlinked
+ // prefix here until the post-unlink directory barrier succeeds avoids both
+ // per-pass allocation and publishing logical removal before it is durable.
+ private final MmapSegment[] trimBatch = new MmapSegment[MAX_TRIMS_PER_RING_PASS];
+ // Test seam: runs after a deferred ring-pass cleanup returns. Null in
+ // production; public sender/pool lifecycle tests use it to observe exact
+ // callback completion without sleeps or polling.
+ private volatile Runnable afterRingCleanupHook;
+ // Test seam: runs at the top of deferUntilWorkerExit, before the
+ // worker-liveness check. Null in production; registration-failure tests
+ // throw from it to simulate an allocation failure (OOM building the
+ // cleanup lambda or growing exitCleanups) while the worker is still
+ // live. Callers must treat such a throw as "worker state unknown",
+ // never as the exact false return meaning the worker loop has exited.
+ private volatile Runnable beforeExitCleanupRegistrationHook;
// Test seam: runs on the worker thread just before the install path's
// synchronized(lock) entry (the one that performs installHotSpare + the
// totalBytes += segmentSize commit). Null in production; tests use it to
@@ -93,33 +117,74 @@ public final class SegmentManager implements QuietCloseable {
// but before ownership/accounting commit. Callers may inject a deregister
// or hold this stale worker snapshot while caller-side cleanup runs.
private volatile Runnable beforeInstallSyncHook;
+ // Test seam: records entry into the per-ring quiescence wait. Null in
+ // production; owned-engine close tests use it to prove they take only the
+ // stronger whole-manager join path, not two sequential timeout budgets.
+ private volatile Runnable beforeRingQuiescenceAwaitHook;
// Test seam: runs on the worker thread just before the trim block's
// synchronized(lock) entry. Null in production; only
// SegmentManagerTrimDeregisterRaceTest installs it, to deterministically
// inject a deregister(ring) call into the exact race window that the
- // registered flag check inside the trim block closes for watermark writes
- // and totalBytes accounting.
+ // entry-state check inside the trim block closes for watermark writes and
+ // totalBytes accounting.
private volatile Runnable beforeTrimSyncHook;
+ // Test seam invoked exactly when a retry transition/recovery is logged.
+ // Null in production; persistent-failure tests use it to prove log bounds
+ // without binding to a particular SLF4J backend.
+ private volatile Runnable retryLogHook;
+ // Entry the worker is claiming or servicing, or null between passes.
+ // Volatile publication lets teardown find the entry after deregister()
+ // removes it from rings. RingEntry.state is the authoritative barrier:
+ // the worker publishes this reference before its REGISTERED->IN_SERVICE
+ // CAS and only touches ring resources after that CAS succeeds.
+ private volatile RingEntry inService;
+ // Cleanup actions handed to the worker's exit block by an owning engine
+ // whose close() found the worker still mid service pass after the bounded
+ // join timed out (see deferUntilWorkerExit). Guarded by {@link #lock};
+ // consumed exactly once by the worker-loop finally, which runs them
+ // OUTSIDE `lock`: they perform syscalls (munmap/unlink/flock release),
+ // and no caller-facing lock may ever be held while running third-party
+ // cleanup code.
+ private ObjList exitCleanups;
+ // A private manager belongs to exactly one CursorSendEngine. Its callback
+ // is preallocated by that engine and stored directly here, so the critical
+ // timed-out-close handoff never allocates an ObjList or grows a backing
+ // array. Guarded by lock and consumed on worker-loop exit outside lock.
+ private Runnable ownedEngineExitCleanup;
private long lastDiskFullLogNs;
private volatile boolean running;
+ // pathScratch free-exactly-once coordination between a timed-out close()
+ // and the worker's exit path. All three are guarded by {@link #lock}.
+ // When close() gives up on the join while the worker loop has not yet
+ // exited, it hands scratch ownership to the worker
+ // (scratchHandedToWorker=true) and the worker frees the buffer in its
+ // exit block; in every other case close() frees it. Without the handoff,
+ // a worker that outlives the bounded join leaks the native scratch
+ // buffer forever, because nobody retries manager cleanup after close()
+ // returns.
+ private boolean scratchFreed;
+ private boolean scratchHandedToWorker;
+ private boolean workerLoopExited;
// Total bytes currently allocated across every segment owned by every
// registered ring (active + sealed + hot-spare). Mutated by the manager
// thread on provision/trim and by register/deregister callers under
// {@link #lock}; the lock covers both paths so the counter stays
// consistent across registration boundaries.
private long totalBytes;
- private long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS;
+ // volatile: read by awaitRingQuiescence() from arbitrary caller threads
+ // while the @TestOnly setter may run on another.
+ private volatile long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS;
// volatile because wakeWorker() reads workerThread without holding the
// monitor; the synchronized start()/close() pair handles the
// start-vs-close ordering.
private volatile Thread workerThread;
public SegmentManager(long segmentSizeBytes) {
- this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES);
+ this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime);
}
public SegmentManager(long segmentSizeBytes, long pollNanos) {
- this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES);
+ this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime);
}
/**
@@ -145,6 +210,22 @@ public SegmentManager(long segmentSizeBytes, long pollNanos) {
* hold an initial active plus one hot spare.
*/
public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) {
+ this(segmentSizeBytes, pollNanos, maxTotalBytes, FilesFacade.INSTANCE, System::nanoTime);
+ }
+
+ @TestOnly
+ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes, FilesFacade filesFacade) {
+ this(segmentSizeBytes, pollNanos, maxTotalBytes, filesFacade, System::nanoTime);
+ }
+
+ @TestOnly
+ public SegmentManager(
+ long segmentSizeBytes,
+ long pollNanos,
+ long maxTotalBytes,
+ FilesFacade filesFacade,
+ LongSupplier ticks
+ ) {
// The pathScratch field initializer has already allocated its native
// buffer by the time this body runs, so a validation throw must free
// it or every failed construction leaks 256 bytes of native memory
@@ -159,9 +240,15 @@ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes)
"maxTotalBytes (" + maxTotalBytes + ") must allow at least one segment of "
+ segmentSizeBytes + " bytes");
}
+ this.filesFacade = filesFacade;
this.segmentSizeBytes = segmentSizeBytes;
this.pollNanos = pollNanos;
this.maxTotalBytes = maxTotalBytes;
+ this.ticks = ticks;
+ }
+
+ FilesFacade filesFacade() {
+ return filesFacade;
}
@Override
@@ -194,18 +281,210 @@ public synchronized void close() {
}
}
if (t.isAlive()) {
- LOG.warn("SegmentManager worker did not stop before close wait completed; "
- + "leaving worker-owned resources allocated");
- return;
+ synchronized (lock) {
+ if (!workerLoopExited) {
+ // Hand pathScratch ownership to the worker: its exit
+ // block frees the buffer under the same lock, so the
+ // native allocation is reclaimed even though this
+ // close() could not confirm termination. workerThread
+ // stays set so isWorkerReaped() reports the incomplete
+ // shutdown and a later close() can retry the join.
+ scratchHandedToWorker = true;
+ LOG.warn("SegmentManager worker did not stop before close wait completed; "
+ + "worker frees its native scratch buffer on exit");
+ return;
+ }
+ }
+ // The worker loop has already run its exit block; the thread
+ // is at most a few instructions from terminating and can no
+ // longer touch manager state. Fall through and reap it.
}
workerThread = null;
}
// Free the rotation-path native scratch buffer only after worker
- // termination has been observed. The worker is the only thread that
- // touches the buffer, but close() uses a bounded join; if the worker is
- // still alive, leaking this one scratch allocation is safer than freeing
- // native memory it may still read or write.
- pathScratch.close();
+ // termination (or worker-loop exit) has been observed. The worker is
+ // the only thread that touches the buffer; the scratchFreed flag
+ // (shared with the worker's exit block) makes the free exactly-once
+ // no matter which side runs last.
+ synchronized (lock) {
+ if (!scratchFreed) {
+ scratchFreed = true;
+ pathScratch.close();
+ }
+ }
+ }
+
+ /**
+ * Hands the single owning engine's preallocated close callback to the
+ * worker exit block. Registration only assigns a reference under
+ * {@link #lock}; it cannot allocate in the timed-out teardown path.
+ * Returns {@code false} only after the worker loop has exited (or when it
+ * never started), so the caller may then clean up inline safely.
+ */
+ public boolean deferOwnedEngineCloseUntilWorkerExit(Runnable cleanup) {
+ synchronized (lock) {
+ if (workerLoopExited || workerThread == null) {
+ return false;
+ }
+ if (ownedEngineExitCleanup != null && ownedEngineExitCleanup != cleanup) {
+ throw new IllegalStateException("owned manager already has an engine-exit cleanup");
+ }
+ ownedEngineExitCleanup = cleanup;
+ return true;
+ }
+ }
+
+ /**
+ * Hands a cleanup action to the current service pass for {@code ring}.
+ * The worker runs it outside the manager lock immediately after that pass
+ * finishes. Returns {@code false} when no pass for the ring remains in
+ * flight, in which case the caller already owns a quiescent ring and must
+ * run the cleanup itself.
+ *
+ * Registration and pass completion coordinate through the entry's atomic
+ * state and callback fields, so there is no gap without an owner: either
+ * this method attaches the cleanup while the pass remains active, the
+ * worker claims an attached cleanup while completing, or this method
+ * observes completed state and rejects the handoff. A repeated registration
+ * for the same pass returns {@code true} without replacing its existing
+ * owner.
+ */
+ public boolean deferUntilRingQuiescent(SegmentRing ring, Runnable cleanup) {
+ RingEntry e = inService;
+ if (e == null || e.ring != ring || !e.isInService()) {
+ return false;
+ }
+ Runnable existing = ENTRY_CLEANUP_UPDATER.get(e);
+ if (existing == null && ENTRY_CLEANUP_UPDATER.compareAndSet(e, null, cleanup)) {
+ if (e.isInService()) {
+ return true;
+ }
+ // Completion changed the state before observing the callback.
+ // Remove our callback and clean inline, unless the worker already
+ // claimed it (in which case that worker remains the owner).
+ return !ENTRY_CLEANUP_UPDATER.compareAndSet(e, cleanup, null);
+ }
+ // A callback already attached to this pass is an owner. The sole
+ // production caller always supplies the engine's preallocated bound
+ // callback; duplicates intentionally do not replace it.
+ return true;
+ }
+
+ /**
+ * Hands a cleanup action to the worker thread's exit block, to run
+ * strictly after the worker loop has finished its final service pass --
+ * i.e. after the last point where the worker can create, write or unlink
+ * anything under a registered ring's slot directory. Returns {@code false}
+ * when the worker loop has already exited (or the worker never started):
+ * the caller must run the cleanup itself, which is equally safe for the
+ * same reason -- no further worker access to the slot is possible.
+ *
+ * This is the slot-ownership transfer used by an owning engine's close()
+ * when the bounded worker join timed out: instead of retiring the slot
+ * until process exit, ring/watermark/flock release moves to the worker,
+ * which is provably the last thread able to touch the slot directory.
+ * The registration here and the exit block's {@code workerLoopExited}
+ * flip share {@link #lock}, so the cleanup runs exactly once: either it
+ * is registered before the flip and the worker runs it, or the flip won
+ * and this method rejects the handoff.
+ *
+ * May throw on allocation failure (the lambda at the call site, the
+ * {@code ObjList}, or its growth). A throw carries NO liveness
+ * information: the worker was never observed, so the caller must treat
+ * it as "worker possibly still live" and retain resources — never as
+ * the exact {@code false} return above.
+ */
+ public boolean deferUntilWorkerExit(Runnable cleanup) {
+ Runnable hook = beforeExitCleanupRegistrationHook;
+ if (hook != null) {
+ hook.run();
+ }
+ synchronized (lock) {
+ if (workerLoopExited || workerThread == null) {
+ return false;
+ }
+ if (exitCleanups == null) {
+ exitCleanups = new ObjList<>();
+ }
+ exitCleanups.add(cleanup);
+ return true;
+ }
+ }
+
+ /**
+ * Quiescence barrier for {@link #deregister(SegmentRing)}. Blocks until
+ * the worker thread is provably no longer executing a service pass for
+ * {@code ring}, or the worker-join timeout elapses. After this returns
+ * {@code true}, the worker will never again touch the ring, its
+ * watermark, or path names under its slot directory: deregister has
+ * removed the entry from the registry, a stale snapshot entry that has
+ * not started its pass is skipped by the registration check at the top
+ * of {@link #serviceRing(RingEntry)}, and this method has observed the
+ * end of any in-flight pass. Only then may the caller release dependent
+ * resources (ring, watermark, segment files, slot lock).
+ *
+ * Returns {@code true} immediately when no worker is running or when
+ * called from the worker thread itself (test hooks inject deregister
+ * calls there; waiting would self-deadlock). Returns {@code false} when
+ * the in-flight pass did not finish within the timeout — the caller must
+ * treat the worker as still live and leak rather than release.
+ *
+ * A pending caller interrupt is preserved but does not abort the wait,
+ * mirroring {@link #close()}.
+ */
+ public boolean awaitRingQuiescence(SegmentRing ring) {
+ Runnable hook = beforeRingQuiescenceAwaitHook;
+ if (hook != null) {
+ hook.run();
+ }
+ Thread t = workerThread;
+ if (t == null || t == Thread.currentThread()) {
+ return true;
+ }
+ long deadlineNanos = System.nanoTime() + workerJoinTimeoutMillis * 1_000_000L;
+ boolean interrupted = Thread.interrupted();
+ try {
+ RingEntry e = inService;
+ if (e == null || e.ring != ring || !e.isInService()) {
+ return true;
+ }
+ synchronized (lock) {
+ e.quiescenceWaiters++;
+ try {
+ while (e.isInService()) {
+ long remainingNanos = deadlineNanos - System.nanoTime();
+ if (remainingNanos <= 0) {
+ return false;
+ }
+ try {
+ // Round up so a sub-millisecond remainder still waits
+ // instead of spinning through wait(0) == wait-forever.
+ lock.wait(Math.max(1L, remainingNanos / 1_000_000L));
+ } catch (InterruptedException ignored) {
+ interrupted = true;
+ }
+ }
+ } finally {
+ e.quiescenceWaiters--;
+ }
+ }
+ return true;
+ } finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ /**
+ * True when no manager worker thread can be running: either
+ * {@link #start()} was never called, or a {@link #close()} confirmed
+ * worker termination and reaped the thread. Owners use this as a
+ * stronger fallback barrier when {@link #awaitRingQuiescence(SegmentRing)}
+ * times out but a subsequent {@code close()} join succeeded.
+ */
+ public synchronized boolean isWorkerReaped() {
+ return workerThread == null;
}
/**
@@ -213,6 +492,13 @@ public synchronized void close() {
* created after this returns, but already-installed spares stay with
* the ring (the ring closes them on its own {@link SegmentRing#close}).
* Idempotent; safe to call from any thread.
+ *
+ * Non-blocking: a worker service pass already in flight for this ring
+ * may still be running when this returns. Callers about to release
+ * resources the worker can reach (the ring itself, its watermark, its
+ * segment files, or the slot lock guarding its directory) MUST follow
+ * up with {@link #awaitRingQuiescence(SegmentRing)} and only release on
+ * a {@code true} result.
*/
public void deregister(SegmentRing ring) {
synchronized (lock) {
@@ -226,7 +512,7 @@ public void deregister(SegmentRing ring) {
// single subtraction covers both the initial seed
// and the net manager activity (provisions minus
// trims) for this ring.
- e.registered = false;
+ e.deregister();
totalBytes -= ring.totalSegmentBytes();
rings.remove(i);
return;
@@ -297,16 +583,36 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) {
wakeWorker();
}
+ @TestOnly
+ public void setAfterRingCleanupHook(Runnable hook) {
+ this.afterRingCleanupHook = hook;
+ }
+
+ @TestOnly
+ public void setBeforeExitCleanupRegistrationHook(Runnable hook) {
+ this.beforeExitCleanupRegistrationHook = hook;
+ }
+
@TestOnly
public void setBeforeInstallSyncHook(Runnable hook) {
this.beforeInstallSyncHook = hook;
}
+ @TestOnly
+ public void setBeforeRingQuiescenceAwaitHook(Runnable hook) {
+ this.beforeRingQuiescenceAwaitHook = hook;
+ }
+
@TestOnly
public void setBeforeTrimSyncHook(Runnable hook) {
this.beforeTrimSyncHook = hook;
}
+ @TestOnly
+ public void setRetryLogHook(Runnable hook) {
+ this.retryLogHook = hook;
+ }
+
@TestOnly
public void setWorkerJoinTimeoutMillis(long millis) {
this.workerJoinTimeoutMillis = millis;
@@ -341,21 +647,19 @@ public void wakeWorker() {
* files in {@code dir}, or {@code -1} if none exist. Skips files that
* don't match the pattern (e.g. the legacy {@code sf-initial.sfa}).
*/
- private static long scanMaxGeneration(String dir) {
+ private long scanMaxGeneration(String dir) {
long max = -1L;
- if (!Files.exists(dir)) return max;
- long find = Files.findFirst(dir);
+ if (!filesFacade.exists(dir)) return max;
+ long find = filesFacade.findFirst(dir);
if (find < 0) {
- LOG.warn("scanMaxGeneration could not enumerate {}; "
- + "next spare may collide with an existing on-disk segment", dir);
- return max;
+ throw new IllegalStateException("could not enumerate SF segment directory " + dir);
}
if (find == 0) return max;
try {
int rc = 1;
while (rc > 0) {
- String name = Files.utf8ToString(Files.findName(find));
- rc = Files.findNext(find);
+ String name = Files.utf8ToString(filesFacade.findName(find));
+ rc = filesFacade.findNext(find);
if (name == null || !name.startsWith("sf-") || !name.endsWith(".sfa")) {
continue;
}
@@ -368,8 +672,11 @@ private static long scanMaxGeneration(String dir) {
// sf-initial.sfa or non-hex — skip
}
}
+ if (rc < 0) {
+ throw new IllegalStateException("could not fully enumerate SF segment directory " + dir);
+ }
} finally {
- Files.findClose(find);
+ filesFacade.findClose(find);
}
return max;
}
@@ -411,7 +718,51 @@ private String nextSparePath(String dir) {
return displayPath;
}
- private void serviceRing(RingEntry e) {
+ private boolean serviceRing(RingEntry e) {
+ // Publish before the CAS so deregister + await can always find the
+ // entry. The state CAS is the ownership decision: if deregister won,
+ // this stale snapshot pass skips the ring without touching it.
+ inService = e;
+ if (!ENTRY_STATE_UPDATER.compareAndSet(e, ENTRY_REGISTERED, ENTRY_IN_SERVICE)) {
+ inService = null;
+ return false;
+ }
+ boolean hasMoreTrimmable;
+ try {
+ hasMoreTrimmable = serviceRing0(e);
+ } finally {
+ e.finishService();
+ Runnable cleanup = e.quiescenceCleanup == null
+ ? null
+ : ENTRY_CLEANUP_UPDATER.getAndSet(e, null);
+ inService = null;
+ // A normal pass performs only the two state CAS operations above.
+ // The manager monitor is entered here only for an actual close
+ // waiter; the recheck under lock prevents lost wakeups.
+ if (e.quiescenceWaiters > 0) {
+ synchronized (lock) {
+ if (e.quiescenceWaiters > 0) {
+ lock.notifyAll();
+ }
+ }
+ }
+ if (cleanup != null) {
+ try {
+ cleanup.run();
+ } catch (Throwable t) {
+ LOG.error("deferred engine cleanup failed after manager-worker ring pass", t);
+ } finally {
+ Runnable hook = afterRingCleanupHook;
+ if (hook != null) {
+ hook.run();
+ }
+ }
+ }
+ }
+ return hasMoreTrimmable;
+ }
+
+ private boolean serviceRing0(RingEntry e) {
// 1. Provision a hot spare if the ring needs one AND we have headroom
// under the disk-total cap. Cap check is per-tick; if we're capped
// here, the ring stays in BACKPRESSURE_NO_SPARE until trim (step 2)
@@ -455,14 +806,21 @@ private void serviceRing(RingEntry e) {
// via its long-ptr overload, bypassing the byte[] + native
// malloc that the String overload would incur on every
// rotation.
- spare = MmapSegment.create(FilesFacade.INSTANCE,
+ spare = MmapSegment.create(filesFacade,
pathScratch.ptr(), path,
- e.ring.nextSeqHint(), segmentSizeBytes);
+ e.ring.nextSeqHint(), segmentSizeBytes, true);
}
Runnable installHook = beforeInstallSyncHook;
if (installHook != null) {
installHook.run();
}
+ if (!memoryMode) {
+ spare.syncHeader();
+ if (filesFacade.fsyncDir(e.dir) != 0) {
+ throw new MmapSegmentException(
+ "could not sync hot-spare directory " + e.dir);
+ }
+ }
// Install + commit atomically under the manager lock.
// If `e.ring` was deregistered between the snapshot
// above and now, abandoning the spare here is the only
@@ -476,7 +834,7 @@ private void serviceRing(RingEntry e) {
// observe the spare in the ring (and subtract it) or
// run before installation (so no install happens).
synchronized (lock) {
- if (e.registered) {
+ if (e.isRegistered()) {
e.ring.installHotSpare(spare);
totalBytes += segmentSizeBytes;
installed = true;
@@ -493,104 +851,302 @@ private void serviceRing(RingEntry e) {
} catch (Throwable ignored) {
}
}
- // Remove the file even when spare is null (i.e. when
- // MmapSegment.create itself threw): MmapSegment.create's
- // catch already best-effort removes, but if anything
- // before mmap (e.g. an exception thrown by the JVM
- // between openCleanRW and the try block) leaves a file
- // on disk, this is the second-line defense. Repeated
- // unlink on an already-removed path is a harmless no-op.
- if (path != null) {
- Files.remove(path);
+ // Only remove the file when the spare object exists, i.e.
+ // MmapSegment.create succeeded and ownership is ours but
+ // installation was rejected (ring deregistered/closed).
+ // When create() itself threw, its catch already removed
+ // anything it put on disk -- and with exclusive create
+ // (O_EXCL) a failure can also mean the path was ALREADY
+ // occupied by a file some other lifecycle owns, which a
+ // blanket unlink here would destroy.
+ if (path != null && spare != null) {
+ filesFacade.remove(path);
}
}
}
}
- // 2. Trim any segments that the ring says are fully acked. For
- // memory-mode rings, "trim" is just close() (Unsafe.free) — no
- // file to unlink.
- //
- // The watermark write and totalBytes commit are registration-gated
- // under `lock` so stale worker snapshots cannot touch the
- // engine-owned watermark or mutate accounting after deregister()
- // returns. drainTrimmable still runs for stale snapshots: it
- // transfers ownership of fully-acked sealed segments to this
- // worker, preserving the old close + unlink behavior.
- //
- // munmap + unlink stay outside the lock — they can be slow
- // and shouldn't block register/deregister or sibling rings.
- ObjList trim;
+ // 2. Trim any segments that the ring says are fully acked. Memory
+ // mode only frees native memory. Disk mode first makes the current
+ // cumulative ACK durable, then durably establishes the watermark's
+ // directory entry, unlinks one bounded batch, and finally commits
+ // those directory removals before ring/accounting state changes.
+ // No syscall runs under the manager lock.
Runnable hook = beforeTrimSyncHook;
if (hook != null) {
hook.run();
}
- synchronized (lock) {
- boolean registered = e.registered;
- // Persist the current ackedFsn watermark BEFORE the trim runs.
- // On host crash between the persist and the unlinks below, the
- // segments survive and the watermark is correct. On crash AFTER
- // the unlinks but before next tick, the segments are gone and
- // the watermark is stale, but recovery clamps with
- // max(lowestSurvivingBaseSeq - 1, watermark) so either ordering
- // is safe. Memory-mode rings (and callers that didn't supply a
- // watermark) skip the write.
- // Persist only on advance to avoid pointless mmap stores when
- // ackedFsn is steady. The store is a single 8-byte put against
- // an already-mapped region -- no syscall, no allocation -- but
- // the gate keeps the dirty-page footprint minimal under
- // steady-state load with no new acks arriving.
- if (registered && e.watermark != null) {
- long currentAck = e.ring.ackedFsn();
- if (currentAck > e.lastPersistedAck) {
- e.watermark.write(currentAck);
- e.lastPersistedAck = currentAck;
- }
- }
- trim = e.ring.drainTrimmable();
- if (registered && trim != null) {
- for (int i = 0, n = trim.size(); i < n; i++) {
- totalBytes -= trim.get(i).sizeBytes();
+ MmapSegment first = e.ring.firstTrimmable();
+ if (first == null) {
+ // Preserve the cheap mmap-only watermark cadence for ACKs that do
+ // not yet cover a complete sealed segment.
+ synchronized (lock) {
+ if (e.isRegistered() && e.watermark != null) {
+ long currentAck = e.ring.ackedFsn();
+ if (currentAck > e.lastPersistedAck) {
+ e.watermark.write(currentAck);
+ e.lastPersistedAck = currentAck;
+ }
}
}
+ return false;
}
- if (trim != null) {
- for (int i = 0, n = trim.size(); i < n; i++) {
- MmapSegment s = trim.get(i);
- String path = s.path();
+
+ if (memoryMode) {
+ int trimmed = 0;
+ while (trimmed < MAX_TRIMS_PER_RING_PASS) {
+ MmapSegment segment = e.ring.firstTrimmable();
+ if (segment == null) {
+ break;
+ }
try {
- s.close();
- if (path != null && !Files.remove(path)) {
- LOG.warn("Failed to unlink trimmed segment {}", path);
+ segment.close();
+ synchronized (lock) {
+ if (e.ring.removeTrimmable(segment)) {
+ trimmed++;
+ if (e.isRegistered()) {
+ totalBytes -= segment.sizeBytes();
+ }
+ }
}
} catch (Throwable t) {
- LOG.warn("Failed to trim segment {}", path == null ? "" : path, t);
+ LOG.warn("Failed to trim memory segment", t);
+ return false;
+ }
+ }
+ return trimmed == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null;
+ }
+
+ // A deferred disk retry does no sync, unlink, or logging work. Signed
+ // subtraction is the standard wrap-safe deadline comparison because
+ // the bounded delay is many orders of magnitude below half the long
+ // range, even when the monotonic clock wraps.
+ long now = ticks.getAsLong();
+ if (e.trimRetryDelayNanos != 0 && now - e.trimRetryAtNanos < 0) {
+ return false;
+ }
+
+ // A disk segment cannot be safely unlinked without durable ACK cover.
+ // The registration check and mmap store are atomic with deregister.
+ // Once the store wins, deregistration may proceed, but its owner must
+ // await this in-service pass before closing the watermark or ring.
+ if (e.watermark == null) {
+ if (!e.missingWatermarkLogged) {
+ e.missingWatermarkLogged = true;
+ LOG.warn("Cannot durably trim acknowledged segments in {} without an ack watermark", e.dir);
+ }
+ return false;
+ }
+ long durableAck;
+ synchronized (lock) {
+ if (!e.isRegistered()) {
+ return false;
+ }
+ durableAck = e.ring.ackedFsn();
+ if (durableAck > e.lastPersistedAck) {
+ e.watermark.write(durableAck);
+ e.lastPersistedAck = durableAck;
+ }
+ }
+ try {
+ e.watermark.sync();
+ if (filesFacade.fsyncDir(e.dir) != 0) {
+ recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, null);
+ return false;
+ }
+ } catch (Throwable t) {
+ recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, t);
+ return false;
+ }
+
+ boolean trimFailed = false;
+ Throwable trimFailure = null;
+ int unlinked = 0;
+ MmapSegment segment = first;
+ long coveredAck = durableAck;
+ while (segment != null && unlinked < MAX_TRIMS_PER_RING_PASS) {
+ long lastSeq = segment.baseSeq() + segment.frameCount() - 1L;
+ if (lastSeq > coveredAck) {
+ break;
+ }
+ MmapSegment next = e.ring.nextSealedAfter(segment);
+ String path = segment.path();
+ try {
+ // Durably commit the head advance past this segment before
+ // unlinking it. The ring recomputes the successor under its
+ // own monitor -- computing it here from `next` would race
+ // with a concurrent rotation sealing the active, letting the
+ // head leapfrog a still-unacked sealed segment (whose file a
+ // later recovery would then discard as "stale below head").
+ e.ring.advanceManifestHeadPast(segment);
+ segment.close();
+ // A retry after a post-unlink directory-sync failure sees an
+ // already absent path. Treat absence as the prior successful
+ // unlink and retry the batch barrier rather than wedging.
+ if (!filesFacade.remove(path) && filesFacade.exists(path)) {
+ trimFailed = true;
+ break;
+ }
+ trimBatch[unlinked++] = segment;
+ segment = next;
+ } catch (Throwable t) {
+ trimFailed = true;
+ trimFailure = t;
+ break;
+ }
+ }
+
+ if (unlinked > 0) {
+ try {
+ if (filesFacade.fsyncDir(e.dir) != 0) {
+ for (int i = 0; i < unlinked; i++) {
+ trimBatch[i] = null;
+ }
+ recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, null);
+ return false;
+ }
+ } catch (Throwable t) {
+ for (int i = 0; i < unlinked; i++) {
+ trimBatch[i] = null;
+ }
+ recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, t);
+ return false;
+ }
+ synchronized (lock) {
+ for (int i = 0; i < unlinked; i++) {
+ MmapSegment removed = trimBatch[i];
+ trimBatch[i] = null;
+ if (e.ring.removeTrimmable(removed) && e.isRegistered()) {
+ totalBytes -= removed.sizeBytes();
+ }
}
}
}
+ if (trimFailed) {
+ recordTrimFailure(e, TRIM_RETRY_UNLINK, now, trimFailure);
+ return false;
+ }
+ recordTrimSuccess(e);
+ return unlinked == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null;
+ }
+
+ private void recordTrimFailure(RingEntry e, int failureKind, long now, Throwable failure) {
+ long delay = e.trimRetryDelayNanos == 0
+ ? TRIM_RETRY_INITIAL_NANOS
+ : Math.min(e.trimRetryDelayNanos << 1, TRIM_RETRY_MAX_NANOS);
+ e.trimRetryAtNanos = now + delay;
+ e.trimRetryDelayNanos = delay;
+ if (e.trimRetryFailureKind != failureKind) {
+ e.trimRetryFailureKind = failureKind;
+ if (failure == null) {
+ LOG.warn("Durable segment trim failed in {} during {} (retry delayed)",
+ e.dir, trimFailureName(failureKind));
+ } else {
+ LOG.warn("Durable segment trim failed in {} during {} (retry delayed)",
+ e.dir, trimFailureName(failureKind), failure);
+ }
+ Runnable hook = retryLogHook;
+ if (hook != null) {
+ hook.run();
+ }
+ }
+ }
+
+ private void recordTrimSuccess(RingEntry e) {
+ if (e.trimRetryDelayNanos != 0) {
+ e.trimRetryAtNanos = 0;
+ e.trimRetryDelayNanos = 0;
+ e.trimRetryFailureKind = TRIM_RETRY_NONE;
+ LOG.info("Durable segment trim recovered in {}", e.dir);
+ Runnable hook = retryLogHook;
+ if (hook != null) {
+ hook.run();
+ }
+ }
+ }
+
+ private static String trimFailureName(int failureKind) {
+ switch (failureKind) {
+ case TRIM_RETRY_PRE_BARRIER:
+ return "covering barrier";
+ case TRIM_RETRY_UNLINK:
+ return "segment unlink";
+ default:
+ return "directory commit barrier";
+ }
}
private void workerLoop() {
- while (running) {
- // Snapshot the rings under the lock so we don't hold it through the
- // (potentially slow) syscalls during creation/unlink. ringSnapshot
- // is a thread-confined field — no per-tick allocation.
- ringSnapshot.clear();
+ try {
+ while (running) {
+ // Snapshot the rings under the lock so we don't hold it through the
+ // (potentially slow) syscalls during creation/unlink. ringSnapshot
+ // is a thread-confined field — no per-tick allocation.
+ ringSnapshot.clear();
+ synchronized (lock) {
+ for (int i = 0, n = rings.size(); i < n; i++) {
+ ringSnapshot.add(rings.getQuick(i));
+ }
+ }
+ boolean hasMoreTrimmable = false;
+ for (int i = 0, n = ringSnapshot.size(); i < n; i++) {
+ if (!running) break;
+ if (serviceRing(ringSnapshot.getQuick(i))) {
+ hasMoreTrimmable = true;
+ }
+ }
+ // Drop strong refs so a deregistered ring becomes collectable
+ // before the next tick (otherwise the snapshot pins it for up
+ // to pollNanos after deregister).
+ ringSnapshot.clear();
+ if (!running) break;
+ if (!hasMoreTrimmable) {
+ LockSupport.parkNanos(pollNanos);
+ }
+ }
+ } finally {
+ // If a timed-out close() abandoned the reap, it handed
+ // pathScratch ownership to this thread (see close()). Freeing it
+ // here reclaims the native buffer even when the worker outlives
+ // every close() attempt — nobody else retries manager cleanup.
+ ObjList cleanups;
+ Runnable ownedEngineCleanup;
synchronized (lock) {
- for (int i = 0, n = rings.size(); i < n; i++) {
- ringSnapshot.add(rings.getQuick(i));
+ workerLoopExited = true;
+ if (scratchHandedToWorker && !scratchFreed) {
+ scratchFreed = true;
+ pathScratch.close();
}
+ cleanups = exitCleanups;
+ exitCleanups = null;
+ ownedEngineCleanup = ownedEngineExitCleanup;
+ ownedEngineExitCleanup = null;
}
- for (int i = 0, n = ringSnapshot.size(); i < n; i++) {
- if (!running) break;
- serviceRing(ringSnapshot.getQuick(i));
+ // Deferred engine cleanups run OUTSIDE
+ // `lock`: they perform syscalls (munmap, unlink, flock release)
+ // and must never execute under a lock that close()/register/
+ // deregister callers contend on. Running them after the loop body
+ // is what makes the handoff safe: this thread can no longer touch
+ // any slot path. They must also never block on a caller-held
+ // monitor — a retried engine.close() joins this thread while
+ // holding the engine monitor, which is why the engine side uses a
+ // lock-free claim (CAS), not synchronization, for exactly-once.
+ if (ownedEngineCleanup != null) {
+ try {
+ ownedEngineCleanup.run();
+ } catch (Throwable t) {
+ LOG.error("deferred owned-engine cleanup failed on manager-worker exit", t);
+ }
+ }
+ if (cleanups != null) {
+ for (int i = 0, n = cleanups.size(); i < n; i++) {
+ try {
+ cleanups.getQuick(i).run();
+ } catch (Throwable t) {
+ LOG.error("deferred engine cleanup failed on manager-worker exit", t);
+ }
+ }
}
- // Drop strong refs so a deregistered ring becomes collectable
- // before the next tick (otherwise the snapshot pins it for up
- // to pollNanos after deregister).
- ringSnapshot.clear();
- if (!running) break;
- LockSupport.parkNanos(pollNanos);
}
}
@@ -607,16 +1163,65 @@ private static final class RingEntry {
// Survives across multiple serviceRing ticks and avoids a
// write-storm when ackedFsn is steady.
long lastPersistedAck = -1L;
- // Guarded by SegmentManager.lock. A worker snapshot may retain this
- // entry after deregister() removes it from rings; registered=false is
- // the O(1) ownership check that prevents post-deregister writes through
- // the engine-owned watermark, hot-spare installs, and accounting.
- boolean registered = true;
+ // Prevents a legacy disk registration without a watermark from
+ // flooding the log on every manager tick.
+ boolean missingWatermarkLogged;
+ // Zero-allocation manager-thread-only retry state. The deadline uses
+ // the manager's monotonic clock and the delay doubles to a fixed cap.
+ long trimRetryAtNanos;
+ long trimRetryDelayNanos;
+ int trimRetryFailureKind;
+ // Updated lock-free by deferUntilRingQuiescent and pass completion.
+ // The field updater avoids one AtomicReference allocation per ring.
+ volatile Runnable quiescenceCleanup;
+ // Waiters mutate this under SegmentManager.lock; pass completion reads
+ // it before taking the otherwise-cold notification path.
+ volatile int quiescenceWaiters;
+ // REGISTERED -> IN_SERVICE -> REGISTERED on a normal pass.
+ // Deregistration changes either registered state to its corresponding
+ // deregistered state; completion then changes DEREGISTERED_IN_SERVICE
+ // to DEREGISTERED. The field updater avoids per-entry allocation.
+ volatile int state = ENTRY_REGISTERED;
RingEntry(SegmentRing ring, String dir, AckWatermark watermark) {
this.ring = ring;
this.dir = dir;
this.watermark = watermark;
}
+
+ void deregister() {
+ while (true) {
+ int current = state;
+ int next = current == ENTRY_IN_SERVICE
+ ? ENTRY_DEREGISTERED_IN_SERVICE
+ : ENTRY_DEREGISTERED;
+ if (current == ENTRY_DEREGISTERED || current == ENTRY_DEREGISTERED_IN_SERVICE
+ || ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) {
+ return;
+ }
+ }
+ }
+
+ void finishService() {
+ while (true) {
+ int current = state;
+ int next = current == ENTRY_DEREGISTERED_IN_SERVICE
+ ? ENTRY_DEREGISTERED
+ : ENTRY_REGISTERED;
+ if (ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) {
+ return;
+ }
+ }
+ }
+
+ boolean isInService() {
+ int current = state;
+ return current == ENTRY_IN_SERVICE || current == ENTRY_DEREGISTERED_IN_SERVICE;
+ }
+
+ boolean isRegistered() {
+ int current = state;
+ return current == ENTRY_REGISTERED || current == ENTRY_IN_SERVICE;
+ }
}
}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
index c8516c4c..4dd24b4c 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
@@ -25,12 +25,15 @@
package io.questdb.client.cutlass.qwp.client.sf.cursor;
import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
import io.questdb.client.std.ObjList;
import io.questdb.client.std.QuietCloseable;
import org.jetbrains.annotations.TestOnly;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.IdentityHashMap;
+
/**
* Chain of {@link MmapSegment}s presented to the user thread as one logical
* append-only log keyed by frame sequence number (FSN). Owns segment
@@ -62,15 +65,25 @@ public final class SegmentRing implements QuietCloseable {
public static final long BACKPRESSURE_NO_SPARE = -1L;
/** Sentinel: append failed because the payload doesn't fit in a fresh segment. */
public static final long PAYLOAD_TOO_LARGE = -2L;
+ private static final RetainedSegmentMembershipMode DEFAULT_MEMBERSHIP_MODE =
+ RetainedSegmentMembershipMode.IDENTITY;
private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class);
+ // Tally of sealed-list entries inspected by nextSealedAfter(). Test-only
+ // operation count for deterministic traversal-complexity assertions.
+ private static long nextSealedComparisons;
// Tally of baseSeq comparisons performed by sortByBaseSeq across every
// openExisting() recovery on this JVM. Used by SegmentRingTest to
// assert the sort stays O(N log N) without relying on wall-clock time
// (CI runner variance makes elapsed-millisecond bounds flaky). Cheap
- // in production: one volatile-free add per partition pass, dwarfed by
- // the mmap I/O the recovery does on every segment.
+ // in production: one volatile-free add per partition pass (plus two per
+ // sift level in the rare heapsort fallback), dwarfed by the mmap I/O
+ // the recovery does on every segment.
private static long sortComparisons;
+ // References copied while compacting the logical sealed-segment head.
+ // Test-only operation count for deterministic trim-complexity assertions.
+ private static long trimMovedReferences;
private final long maxBytesPerSegment;
+ private final SfManifest manifest;
// Sealed segments in baseSeq order, oldest first. Active is held separately.
// Single-writer (producer thread, on rotation); single-reader at trim time
// (the segment manager). For now, both sides synchronize via the single-
@@ -78,6 +91,9 @@ public final class SegmentRing implements QuietCloseable {
// looks at sealedSegments after observing a higher ackedFsn, by which
// point the producer thread's add to sealedSegments has retired.
private final ObjList sealedSegments = new ObjList<>();
+ // Logical head of sealedSegments. Head removal nulls one entry and advances
+ // this index; occasional compaction bounds unused prefix slots.
+ private int sealedHead;
// High-water byte offset within the active segment at which we proactively
// ask the segment manager to provision a spare (if one isn't already
// installed). Computed once as 3/4 of segment capacity -- leaves the manager
@@ -120,10 +136,15 @@ public final class SegmentRing implements QuietCloseable {
* frameCount == 0); typically supplied by the segment manager at startup.
*/
public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) {
+ this(initialActive, maxBytesPerSegment, null);
+ }
+
+ SegmentRing(MmapSegment initialActive, long maxBytesPerSegment, SfManifest manifest) {
if (initialActive == null) {
throw new IllegalArgumentException("initialActive must not be null");
}
this.active = initialActive;
+ this.manifest = manifest;
this.maxBytesPerSegment = maxBytesPerSegment;
// 3/4 of capacity gives the manager a full quarter-segment of producer
// runway before backpressure kicks in. Long math, no float, no alloc.
@@ -133,190 +154,610 @@ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) {
}
/**
- * Recovers a ring from segments already on disk in {@code sfDir}. Used at
- * sender startup when the user's previous session left durable but
- * not-yet-acked frames behind. Walks every {@code *.sfa} file in the
- * directory, opens each via {@link MmapSegment#openExisting}, and
- * arranges them by baseSeq:
- *
- *
Highest-baseSeq segment becomes the active (further appends land
- * there until it fills, at which point normal rotation kicks in).
- *
All others become sealed segments awaiting ACK and trim.
- *
- * Returns {@code null} if the directory is empty or contains no
- * recognizable {@code .sfa} files -- the caller should then construct a
- * fresh ring with {@link #SegmentRing(MmapSegment, long)} and a freshly
- * created initial segment.
- *
- * Recovery is best-effort: a single bad-magic file is silently skipped
- * (logged-then-ignored is the right call here; a stray unrelated file in
- * the SF dir shouldn't take the whole sender down). A failure to open
- * an otherwise-valid segment IS fatal -- the caller's data integrity
- * depends on every segment being readable.
+ * Compatibility wrapper using the production facade. New startup code uses
+ * {@link #recover(FilesFacade, String, long)} so EMPTY is explicit.
*/
public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
- if (!Files.exists(sfDir)) {
- return null;
+ return openExisting(FilesFacade.INSTANCE, sfDir, maxBytesPerSegment);
+ }
+
+ /**
+ * Facade-aware variant of {@link #openExisting(String, long)}: every
+ * filesystem touch (enumeration, open, mmap, quarantine rename, manifest
+ * I/O) goes through {@code filesFacade} so recovery's fail-closed behavior
+ * can be fault-injected in tests. Returns {@code null} when the slot holds
+ * nothing recoverable; throws {@link MmapSegmentException} when the slot's
+ * state cannot be proven safe.
+ */
+ public static SegmentRing openExisting(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) {
+ Recovery recovery = recover(filesFacade, sfDir, maxBytesPerSegment);
+ return recovery.status == RecoveryStatus.EMPTY ? null : recovery.ring;
+ }
+
+ /**
+ * Exhaustively discovers and validates an SF slot without mutation. Only
+ * after enumeration, opens, CRC scans, contiguity and manifest boundaries
+ * all succeed does it migrate a legacy chain or discard validated spares.
+ */
+ static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) {
+ return recover(filesFacade, sfDir, maxBytesPerSegment, null);
+ }
+
+ static Recovery recover(
+ FilesFacade filesFacade,
+ String sfDir,
+ long maxBytesPerSegment,
+ MembershipObserver membershipObserver
+ ) {
+ if (!filesFacade.exists(sfDir)) {
+ return Recovery.empty();
}
- ObjList opened = new ObjList<>();
- long find = Files.findFirst(sfDir);
+ ObjList names = new ObjList<>();
+ long find = filesFacade.findFirst(sfDir);
if (find < 0) {
- LOG.warn("openExisting could not enumerate {} - treating as empty, "
- + "but this may indicate a permission or transient error", sfDir);
- return null;
- }
- if (find == 0) {
- return null;
+ throw new MmapSegmentException("could not enumerate SF directory " + sfDir);
}
- // Outer try-catch: anything escaping the recovery body -- IOOBE from
- // ObjList growth, OOM from native mmap during MmapSegment.openExisting,
- // unforeseen RuntimeException from the contiguity check, etc. -- must
- // not leave fds + mmaps owned by `opened` orphaned. Close every
- // recovered segment and rethrow so the engine surfaces the failure.
- try {
+ if (find > 0) {
+ int rc = 1;
try {
- int rc = 1;
while (rc > 0) {
- String name = Files.utf8ToString(Files.findName(find));
+ String name = Files.utf8ToString(filesFacade.findName(find));
if (name != null && name.endsWith(".sfa")) {
- String path = sfDir + "/" + name;
- MmapSegment seg = null;
- try {
- seg = MmapSegment.openExisting(path);
- // Filter out empty leftovers -- typically hot-spare
- // segments the manager pre-allocated for a prior
- // session that never got rotated into active. They
- // carry the provisional baseSeq=0 and frameCount=0,
- // which would otherwise collide with the real
- // baseSeq=0 segment and trip the contiguity check
- // below. No data to recover; close and unlink.
- // Without the unlink the file persists across crash
- // cycles and the disk leak compounds with every
- // unclean shutdown.
- //
- // CAUTION: only unlink when the file is genuinely
- // empty past the header. If frame[0] failed CRC
- // (bit-rot, partial-page-write at crash, etc.) but
- // valid frames followed, scanFrames returns
- // lastGood=HEADER_SIZE and frameCount=0 -- yet
- // tornTailBytes is non-zero. Treating that as
- // "empty hot-spare" would silently destroy every
- // surviving frame. Quarantine to .corrupt
- // instead so a postmortem can recover what's left.
- if (seg.frameCount() == 0) {
- long torn = seg.tornTailBytes();
- seg.close();
- seg = null;
- if (torn > 0) {
- Files.rename(path, path + ".corrupt");
- } else {
- Files.remove(path);
- }
- } else {
- opened.add(seg);
- seg = null;
+ names.add(name);
+ }
+ rc = filesFacade.findNext(find);
+ }
+ if (rc < 0) {
+ throw new MmapSegmentException("could not fully enumerate SF directory " + sfDir);
+ }
+ } finally {
+ filesFacade.findClose(find);
+ }
+ }
+
+ ObjList all = new ObjList<>();
+ // Files whose own bytes prove corruption (bad magic, sub-header size,
+ // negative baseSeq, unreadable header page). They are excluded from
+ // the chain and quarantined to .corrupt — but only AFTER the
+ // surviving chain validates (or resolves to EMPTY), so a failed
+ // recovery never mutates the slot. Whether a quarantined file was
+ // load-bearing is decided by the manifest-boundary / contiguity
+ // checks below, not by the skip itself. Operational open errors
+ // (EMFILE, EACCES, mmap rejection, unsupported version) are NOT in
+ // this bucket: they throw the plain MmapSegmentException type and
+ // abort recovery, because the underlying file may be perfectly
+ // intact and silently dropping it could lose durable frames.
+ ObjList corruptPaths = null;
+ SfManifest manifest = null;
+ try {
+ for (int i = 0, n = names.size(); i < n; i++) {
+ String path = sfDir + "/" + names.get(i);
+ try {
+ all.add(MmapSegment.openExisting(filesFacade, path));
+ } catch (MmapSegmentCorruptionException e) {
+ LOG.warn("recovery: {} is not a readable SF segment; excluding it and "
+ + "deferring quarantine until the surviving chain validates -- {}",
+ path, e.toString());
+ if (corruptPaths == null) {
+ corruptPaths = new ObjList<>();
+ }
+ corruptPaths.add(path);
+ } catch (MmapSegmentException e) {
+ throw new MmapSegmentException("recovery failed for recognized segment " + path, e);
+ }
+ }
+ manifest = SfManifest.open(filesFacade, sfDir);
+ if (all.size() == 0) {
+ if (corruptPaths != null) {
+ // Nothing valid survived. With a manifest this is still a
+ // hole we can prove (boundaries reference segments that are
+ // now unreadable) -- fail without mutating. Without one,
+ // legacy semantics apply: quarantine and start fresh.
+ if (manifest != null) {
+ throw new MmapSegmentException("every SF segment in " + sfDir
+ + " is corrupt but " + SfManifest.FILE_NAME
+ + " references durable data");
+ }
+ quarantineCorrupt(filesFacade, corruptPaths);
+ return Recovery.empty();
+ }
+ if (manifest != null) {
+ // No .sfa files at all. Two legitimate protocols produce
+ // this: the close-time drain unlinks the last segment
+ // before it removes the manifest, and a fresh-start crash
+ // can leave a boundary-less (0,0) manifest behind. In
+ // both cases nothing recoverable exists, so accept EMPTY
+ // -- but shout, because a manual wipe of segment files
+ // looks identical and the operator should know.
+ LOG.warn("SF manifest exists in {} with no segment files "
+ + "(clean-drain or fresh-start crash window, or manual "
+ + "segment removal); discarding it and starting fresh", sfDir);
+ manifest.close();
+ manifest = null;
+ if (!SfManifest.removeFile(filesFacade, sfDir)) {
+ throw new MmapSegmentException(
+ "could not remove stale SF manifest in " + sfDir);
+ }
+ }
+ return Recovery.empty();
+ }
+
+ ObjList data = new ObjList<>();
+ boolean requiresManifest = false;
+ for (int i = 0, n = all.size(); i < n; i++) {
+ MmapSegment segment = all.get(i);
+ requiresManifest |= segment.manifestRequired();
+ if (segment.frameCount() > 0) {
+ data.add(segment);
+ }
+ }
+ sortByBaseSeq(data, 0, data.size());
+ if (manifest == null && requiresManifest) {
+ throw new MmapSegmentException("new-format SF segment exists but "
+ + SfManifest.FILE_NAME + " is missing");
+ }
+
+ MmapSegment active;
+ ObjList chain = new ObjList<>();
+ long headBase;
+ long activeBase;
+ if (manifest == null) {
+ if (data.size() > 0) {
+ validateContiguous(data);
+ for (int i = 0, n = data.size(); i < n; i++) {
+ chain.add(data.get(i));
+ }
+ active = chain.get(chain.size() - 1);
+ headBase = chain.get(0).baseSeq();
+ activeBase = active.baseSeq();
+ } else {
+ active = chooseEmptyInitial(all, sfDir);
+ if (active == null) {
+ // Legacy slot holding only empty leftovers, every one
+ // of them torn (a clean empty would have been chosen).
+ // Nothing recoverable: quarantine the torn evidence,
+ // drop the clean debris, start fresh -- exactly the
+ // pre-manifest behavior.
+ for (int i = 0, n = all.size(); i < n; i++) {
+ MmapSegment segment = all.get(i);
+ String path = segment.path();
+ long torn = segment.tornTailBytes();
+ segment.close();
+ if (torn > 0) {
+ quarantineFile(filesFacade, path);
+ } else if (!filesFacade.remove(path)) {
+ LOG.warn("could not remove empty SF leftover {}", path);
}
- } catch (MmapSegmentException t) {
- // Per-file data error (bad magic, bad header,
- // unsupported version, mmap rejection on this one
- // file). Don't take down recovery for one corrupt
- // .sfa -- log and skip so siblings still recover.
- // Resource exhaustion (OOM) and programmer errors
- // (IOOBE) deliberately propagate to the outer
- // catch, which closes every already-recovered
- // segment and rethrows: continuing the loop after
- // an OOM would just fail again on the next file
- // while silently leaking the segments we managed
- // to recover before it.
- LOG.warn("openExisting: skipping {} -- {}", path, t.toString());
- } finally {
- // Close any seg whose ownership wasn't transferred
- // (either to opened, or via the empty-branch close
- // above). Fires on a propagating throw between
- // open and transfer -- most importantly an OOM
- // from ObjList.add growing its backing array
- // after the mmap+fd were already acquired.
- if (seg != null) {
- try {
- seg.close();
- } catch (Throwable closeErr) {
- LOG.warn("openExisting: error closing in-flight segment {}",
- path, closeErr);
- }
+ }
+ all.clear();
+ quarantineCorrupt(filesFacade, corruptPaths);
+ return Recovery.empty();
+ }
+ chain.add(active);
+ headBase = active.baseSeq();
+ activeBase = active.baseSeq();
+ }
+ manifest = SfManifest.create(filesFacade, sfDir, headBase, activeBase);
+ for (int i = 0, n = chain.size(); i < n; i++) {
+ chain.get(i).markManifestRequired();
+ }
+ } else {
+ headBase = manifest.headBase();
+ activeBase = manifest.activeBase();
+ for (int i = 0, n = data.size(); i < n; i++) {
+ MmapSegment segment = data.get(i);
+ long end = segment.baseSeq() + segment.frameCount();
+ if (segment.baseSeq() < headBase) {
+ if (end > headBase) {
+ throw new MmapSegmentException("segment overlaps committed SF head boundary");
+ }
+ continue; // acknowledged stale file after manifest-before-unlink crash
+ }
+ if (segment.baseSeq() > activeBase) {
+ throw new MmapSegmentException("segment exists beyond committed SF active boundary");
+ }
+ chain.add(segment);
+ }
+ if (chain.size() > 0) {
+ validateContiguous(chain);
+ if (chain.get(0).baseSeq() != headBase) {
+ throw new MmapSegmentException("missing expected SF head segment at base " + headBase);
+ }
+ }
+ active = findActive(all, activeBase);
+ if (active == null) {
+ if (chain.size() == 0 && headBase == activeBase && corruptPaths == null) {
+ // Clean-drain crash window: the close-time drain first
+ // durably collapses the boundaries to head == active
+ // (declaring every frame acked), then unlinks segments
+ // in ascending order -- so dying between the active's
+ // unlink and the spare's/manifest's leaves exactly
+ // this state: no data frame anywhere, no file at the
+ // committed active base, only empty spares and/or
+ // acked stale files. Nothing recoverable exists;
+ // accept EMPTY and clear the debris. Guarded on
+ // corruptPaths because an unreadable .sfa of unknown
+ // identity could be the real active -- in that case
+ // fail closed instead of guessing.
+ LOG.warn("SF manifest in {} has collapsed boundaries ({}) with no "
+ + "segment at the active base and no recovered frames; "
+ + "accepting the clean-drain crash window as empty",
+ sfDir, activeBase);
+ for (int i = 0, n = all.size(); i < n; i++) {
+ MmapSegment segment = all.get(i);
+ String path = segment.path();
+ long torn = segment.tornTailBytes();
+ segment.close();
+ if (torn > 0) {
+ quarantineFile(filesFacade, path);
+ } else if (!filesFacade.remove(path)) {
+ LOG.warn("could not remove drained SF leftover {}", path);
}
}
+ all.clear();
+ manifest.close();
+ manifest = null;
+ if (!SfManifest.removeFile(filesFacade, sfDir)) {
+ throw new MmapSegmentException(
+ "could not remove stale SF manifest in " + sfDir);
+ }
+ return Recovery.empty();
}
- rc = Files.findNext(find);
+ throw new MmapSegmentException("missing expected SF active segment at base " + activeBase);
}
- } finally {
- Files.findClose(find);
- }
- if (opened.size() == 0) {
- return null;
- }
- // Sort by baseSeq ascending. Worst-case segment count is
- // sf_max_total_bytes / sf_max_bytes -- at the documented ceiling
- // (1 TiB / 64 MiB) that is ~16K entries, where an O(N²) sort spends
- // multiple seconds in compares + shifts before the I/O thread can
- // start. In-place quicksort with median-of-three pivot keeps the
- // no-allocation discipline of the surrounding code; median-of-three
- // is required because readdir on many filesystems returns entries
- // in lexicographic (== baseSeq-hex) order and a naive first-element
- // pivot would degrade back to O(N²) on exactly that common case.
- sortByBaseSeq(opened, 0, opened.size());
- // Sanity: the recovered segments must form a contiguous FSN range.
- // Detect gaps so they don't silently produce duplicate or missing
- // FSNs after recovery. A gap means a segment went missing (a
- // manual deletion) or a sealed segment under-recovered -- its tail
- // was cut short by a sparse/unbacked page or a mid-file media error
- // (bad sector), the same class of fault scanFrames tolerates on the
- // active segment but which corrupts the range on a sealed one.
- for (int i = 1, n = opened.size(); i < n; i++) {
- MmapSegment prev = opened.get(i - 1);
- MmapSegment curr = opened.get(i);
- long expected = prev.baseSeq() + prev.frameCount();
- if (curr.baseSeq() != expected) {
- throw new MmapSegmentException(
- "FSN gap in recovered segments: prev baseSeq=" + prev.baseSeq()
- + " frameCount=" + prev.frameCount()
- + " expected next baseSeq=" + expected
- + " but got " + curr.baseSeq()
- + " -- a segment was deleted, or a sealed segment's tail was"
- + " truncated (sparse/unbacked page or disk media error);"
- + " check disk health");
+ if (chain.size() == 0) {
+ if (headBase != activeBase || active.frameCount() != 0 || corruptPaths != null) {
+ // corruptPaths guard: with an unreadable .sfa in the
+ // slot, the innocent-looking empty at the active base
+ // could be a leftover spare coincidentally carrying
+ // the same provisional baseSeq as a corrupted real
+ // active -- accepting it would quarantine unacked
+ // frames and re-issue their FSNs. Fail closed.
+ throw new MmapSegmentException(
+ "missing SF chain between committed boundaries"
+ + (corruptPaths != null
+ ? " (a corrupt segment prevents proving the empty state)" : ""));
+ }
+ chain.add(active);
+ } else if (chain.get(chain.size() - 1) != active) {
+ MmapSegment tail = chain.get(chain.size() - 1);
+ long chainEnd = tail.baseSeq() + tail.frameCount();
+ if (corruptPaths == null && active.frameCount() == 0 && active.baseSeq() == chainEnd) {
+ // Rotation committed (manifest fsync'd, promoted spare's
+ // header synced) but the process/OS died before a single
+ // frame of the new active reached disk: the sealed chain
+ // ends exactly where the empty active begins. Legal
+ // crash state -- resume appending into it. Refused when
+ // corrupt segments exist (same stand-in hazard as the
+ // empty-chain acceptance above).
+ chain.add(active);
+ } else {
+ throw new MmapSegmentException(
+ "missing expected SF active/tail segment at base " + activeBase);
+ }
+ }
+ for (int i = 0, n = chain.size() - 1; i < n; i++) {
+ if (chain.get(i).tornTailBytes() > 0) {
+ throw new MmapSegmentException("corrupt torn tail in sealed SF segment " + chain.get(i).path());
+ }
+ }
+ for (int i = 0, n = chain.size(); i < n; i++) {
+ chain.get(i).markManifestRequired();
+ }
+ }
+
+ // Build membership and clean validated extras while every mmap and
+ // the manifest still have one local owner. In particular, an OOM
+ // while allocating the identity map falls through to the outer
+ // failure cleanup instead of stranding extras after transfer.
+ RetainedSegmentMembership retained = newDefaultMembership(chain, membershipObserver);
+ for (int i = 0, n = all.size(); i < n; i++) {
+ MmapSegment segment = all.get(i);
+ if (!retained.contains(segment)) {
+ String path = null;
+ long torn = 0;
+ Throwable inspectionError = null;
+ try {
+ path = segment.path();
+ torn = segment.tornTailBytes();
+ } catch (Throwable error) {
+ inspectionError = error;
+ }
+ // A close failure is not best-effort: keep local ownership
+ // and fail through the outer cleanup rather than transfer a
+ // ring while an extra may still own an mmap or fd.
+ segment.close();
+ all.setQuick(i, null);
+ if (inspectionError != null) {
+ warnPostRecoveryCleanupFailure(inspectionError);
+ continue;
+ }
+ try {
+ cleanupClosedExtra(filesFacade, path, torn);
+ } catch (Throwable cleanupError) {
+ warnPostRecoveryCleanupFailure(cleanupError);
+ }
}
}
- // The newest segment becomes the active. Even if it's full, that's OK:
- // the next appendOrFsn returns BACKPRESSURE_NO_SPARE, the manager
- // installs a hot spare, the producer rotates. Same fast path as a
- // mid-life ring.
- int last = opened.size() - 1;
- MmapSegment active = opened.get(last);
- opened.remove(last);
- SegmentRing ring = new SegmentRing(active, maxBytesPerSegment);
- // Older segments become sealed in baseSeq order.
- for (int i = 0, n = opened.size(); i < n; i++) {
- ring.sealedSegments.add(opened.get(i));
+ quarantineCorrupt(filesFacade, corruptPaths);
+
+ for (int i = 1, n = chain.size(); i < n; i++) {
+ chain.get(i - 1).linkSuccessor(chain.get(i));
}
- return ring;
+ SegmentRing ring = new SegmentRing(active, maxBytesPerSegment, manifest);
+ for (int i = 0, n = chain.size() - 1; i < n; i++) {
+ ring.sealedSegments.add(chain.get(i));
+ }
+ // Allocate the return wrapper before transfer. Until this succeeds,
+ // the outer catch remains the sole owner and can close all segments
+ // and the manifest if construction or sealed-list growth fails.
+ Recovery recovery = Recovery.recovered(ring);
+ manifest = null;
+ all.clear();
+ return recovery;
} catch (Throwable t) {
- // Close every recovered MmapSegment that's still in `opened`.
- // After the success path, `opened` no longer contains the active
- // segment (removed above), but the sealed segments transferred to
- // ring.sealedSegments are still owned by the ring once it's
- // returned -- so this catch only fires before the return statement.
- for (int i = 0, n = opened.size(); i < n; i++) {
+ for (int i = 0, n = all.size(); i < n; i++) {
+ MmapSegment segment = all.get(i);
+ if (segment != null) {
+ try {
+ segment.close();
+ } catch (Throwable closeError) {
+ warnRecoveryCloseFailure(closeError);
+ }
+ }
+ }
+ if (manifest != null) {
try {
- opened.get(i).close();
- } catch (Throwable closeErr) {
- LOG.warn("openExisting: error closing recovered segment during cleanup",
- closeErr);
+ manifest.close();
+ } catch (Throwable closeError) {
+ warnRecoveryCloseFailure(closeError);
}
}
throw t;
}
}
+ /**
+ * Durably advances the manifest head past {@code trimming} (the sealed
+ * segment the manager is about to unlink). The successor and the current
+ * active are both read under the ring monitor, so a concurrent rotation
+ * (which also mutates the manifest under this monitor) can never make the
+ * head leapfrog a still-live sealed segment: if rotation sealed the old
+ * active after the caller's snapshot, {@code trimming.successor()} now
+ * points at that sealed segment, not at the new active.
+ */
+ synchronized void advanceManifestHeadPast(MmapSegment trimming) {
+ if (manifest == null) {
+ return;
+ }
+ MmapSegment successor = trimming.successor();
+ long newHeadBase = (successor == null || successor == active)
+ ? active.baseSeq()
+ : successor.baseSeq();
+ manifest.update(newHeadBase, active.baseSeq());
+ }
+
+ /**
+ * Picks the clean (untorn) empty segment to reuse as a legacy slot's
+ * initial active, preferring {@code sf-initial.sfa}. Returns {@code null}
+ * when no clean empty exists; torn empties are never reused here because
+ * their bytes are quarantine evidence, not blank space.
+ */
+ private static MmapSegment chooseEmptyInitial(ObjList all, String sfDir) {
+ String initialPath = sfDir + "/sf-initial.sfa";
+ MmapSegment selected = null;
+ for (int i = 0, n = all.size(); i < n; i++) {
+ MmapSegment segment = all.get(i);
+ if (segment.frameCount() != 0 || segment.tornTailBytes() > 0) {
+ continue;
+ }
+ if (selected == null || initialPath.equals(segment.path())) {
+ selected = segment;
+ }
+ }
+ return selected;
+ }
+
+ private static void cleanupClosedExtra(FilesFacade filesFacade, String path, long torn) {
+ if (torn > 0) {
+ quarantineFile(filesFacade, path);
+ } else if (!filesFacade.remove(path)) {
+ LOG.warn("could not remove validated stale/empty SF segment {}", path);
+ }
+ }
+
+ private static RetainedSegmentMembership newDefaultMembership(
+ final ObjList chain,
+ final MembershipObserver observer
+ ) {
+ if (observer != null) {
+ observer.beforeMembershipAllocation();
+ }
+ if (DEFAULT_MEMBERSHIP_MODE == RetainedSegmentMembershipMode.LINEAR) {
+ if (observer == null) {
+ return new RetainedSegmentMembership() {
+ @Override
+ public boolean contains(MmapSegment segment) {
+ for (int i = 0, n = chain.size(); i < n; i++) {
+ if (chain.get(i) == segment) {
+ return true;
+ }
+ }
+ return false;
+ }
+ };
+ }
+ return new RetainedSegmentMembership() {
+ @Override
+ public boolean contains(MmapSegment segment) {
+ for (int i = 0, n = chain.size(); i < n; i++) {
+ observer.onMembershipOperation();
+ if (chain.get(i) == segment) {
+ return true;
+ }
+ }
+ return false;
+ }
+ };
+ }
+
+ final IdentityHashMap retained = new IdentityHashMap<>(chain.size());
+ for (int i = 0, n = chain.size(); i < n; i++) {
+ retained.put(chain.get(i), Boolean.TRUE);
+ }
+ if (observer == null) {
+ return new RetainedSegmentMembership() {
+ @Override
+ public boolean contains(MmapSegment segment) {
+ return retained.containsKey(segment);
+ }
+ };
+ }
+ return new RetainedSegmentMembership() {
+ @Override
+ public boolean contains(MmapSegment segment) {
+ observer.onMembershipOperation();
+ return retained.containsKey(segment);
+ }
+ };
+ }
+
+ /** Renames every collected corrupt path to {@code .corrupt}, best-effort. */
+ private static void quarantineCorrupt(FilesFacade filesFacade, ObjList corruptPaths) {
+ if (corruptPaths == null) {
+ return;
+ }
+ for (int i = 0, n = corruptPaths.size(); i < n; i++) {
+ try {
+ quarantineFile(filesFacade, corruptPaths.get(i));
+ } catch (Throwable cleanupError) {
+ warnPostRecoveryCleanupFailure(cleanupError);
+ }
+ }
+ }
+
+ private static void quarantineFile(FilesFacade filesFacade, String path) {
+ if (filesFacade.rename(path, path + ".corrupt") != 0) {
+ LOG.warn("could not quarantine {} to {}.corrupt; it will be re-examined "
+ + "on the next recovery", path, path);
+ }
+ }
+
+ /**
+ * Locates the segment the manifest's {@code activeBase} refers to.
+ * Preference order among same-base candidates:
+ *
+ *
a segment with recovered frames (the durable chain tail);
+ *
an empty segment with a torn tail (the promoted active whose
+ * first frame write was cut short — an attempted write marks it as
+ * the one rotation actually exposed);
+ *
a clean empty segment.
+ *
+ * Multiple equivalent empties at the same base are NOT an error: a fresh
+ * start or a rotation crash routinely leaves both the initial/promoted
+ * segment and a provisioned hot spare carrying the same provisional
+ * baseSeq. They are interchangeable blanks — pick one deterministically
+ * and let the extras cleanup discard the rest. Bricking startup on this
+ * state would turn every "kill -9 shortly after start" into a manual
+ * repair.
+ */
+ private static MmapSegment findActive(ObjList all, long activeBase) {
+ MmapSegment tornEmpty = null;
+ MmapSegment cleanEmpty = null;
+ for (int i = 0, n = all.size(); i < n; i++) {
+ MmapSegment segment = all.get(i);
+ if (segment.baseSeq() != activeBase) {
+ continue;
+ }
+ if (segment.frameCount() > 0) {
+ return segment;
+ }
+ if (segment.tornTailBytes() > 0) {
+ if (tornEmpty == null) {
+ tornEmpty = segment;
+ }
+ } else if (cleanEmpty == null) {
+ cleanEmpty = segment;
+ }
+ }
+ return tornEmpty != null ? tornEmpty : cleanEmpty;
+ }
+
+ private static void validateContiguous(ObjList segments) {
+ for (int i = 1, n = segments.size(); i < n; i++) {
+ MmapSegment previous = segments.get(i - 1);
+ MmapSegment current = segments.get(i);
+ long expected = previous.baseSeq() + previous.frameCount();
+ if (current.baseSeq() != expected) {
+ throw new MmapSegmentException("FSN gap in recovered segments: expected "
+ + expected + " but got " + current.baseSeq());
+ }
+ }
+ }
+
+ private static void warnPostRecoveryCleanupFailure(Throwable cleanupError) {
+ try {
+ LOG.warn("post-recovery cleanup failed; leftover files will be "
+ + "re-examined on the next startup", cleanupError);
+ } catch (Throwable ignored) {
+ // Cleanup diagnostics must not invalidate a recovered ring.
+ }
+ }
+
+ private static void warnRecoveryCloseFailure(Throwable closeError) {
+ try {
+ LOG.warn("error closing SF resource after recovery failure", closeError);
+ } catch (Throwable ignored) {
+ // Preserve the original recovery failure and continue closing.
+ }
+ }
+
+ interface MembershipObserver {
+ void beforeMembershipAllocation();
+
+ void onMembershipOperation();
+ }
+
+ static final class Recovery {
+ private final SegmentRing ring;
+ private final RecoveryStatus status;
+
+ private Recovery(RecoveryStatus status, SegmentRing ring) {
+ this.status = status;
+ this.ring = ring;
+ }
+
+ static Recovery empty() {
+ return new Recovery(RecoveryStatus.EMPTY, null);
+ }
+
+ SegmentRing ring() {
+ return ring;
+ }
+
+ static Recovery recovered(SegmentRing ring) {
+ return new Recovery(RecoveryStatus.RECOVERED, ring);
+ }
+
+ RecoveryStatus status() {
+ return status;
+ }
+ }
+
+ enum RecoveryStatus {
+ EMPTY,
+ RECOVERED
+ }
+
+ interface RetainedSegmentMembership {
+ boolean contains(MmapSegment segment);
+ }
+
+ private enum RetainedSegmentMembershipMode {
+ IDENTITY,
+ LINEAR
+ }
+
/**
* Highest FSN that the server has ACK'd. Read by the segment manager to
* decide which sealed segments are safe to munmap + unlink.
@@ -379,15 +820,52 @@ public long appendOrFsn(long payloadAddr, int payloadLen) {
// full, so its frameCount is stable, and (b) the spare hasn't been
// appended to yet (rebaseSeq enforces that). The segment manager's
// earlier guess at baseSeq is irrelevant.
- long actualBase = active.baseSeq() + active.frameCount();
+ MmapSegment previous = active;
+ long actualBase = previous.baseSeq() + previous.frameCount();
spare.rebaseSeq(actualBase);
- // Mutate sealedSegments under the same monitor used by
- // snapshotSealedSegments -- the I/O thread reads through that
- // path and must not see a half-resized ObjList.
+ if (manifest != null) {
+ // Make the spare's rebased identity durable BEFORE the manifest
+ // references it. Without this barrier an OS crash could leave a
+ // durable manifest pointing at baseSeq=actualBase while the
+ // spare's on-disk header still carries the manager's
+ // provisional guess -- recovery would then find no segment at
+ // the committed active boundary and fail a startup that lost
+ // nothing. One msync per rotation, amortized over a whole
+ // segment of appends; runs outside the monitor because the
+ // spare is not yet visible to any other thread.
+ //
+ // Deliberately NOT msync'd here: the sealed predecessor's
+ // data pages. A power loss can therefore tear the sealed
+ // tail after the boundary is committed, and recovery will
+ // fail closed on chainEnd != activeBase. That is the
+ // intended semantics -- page-level durability of frame data
+ // follows the sender's opt-in msync cadence, and recovery
+ // must refuse to guess when the two disagree.
+ spare.syncHeader();
+ }
+ // Publish the successor before the volatile active promotion. The
+ // same monitor protects the sealed list and nextSealedAfter's trim
+ // fallback, while the volatile link also remains readable from a
+ // current segment after the manager removes and closes it.
synchronized (this) {
- sealedSegments.add(active);
+ if (manifest != null) {
+ // Inside the monitor: serialized with the trim path's
+ // advanceManifestHeadPast so neither writer publishes a
+ // boundary computed from a state the other has already
+ // moved past (SfManifest additionally clamps monotonic).
+ // BEFORE any ring mutation: if the manifest fsync throws,
+ // the rotation never happened -- previous stays active,
+ // the spare stays installed, and the producer's retry
+ // re-runs this block from a consistent state.
+ long headBase = sealedHead < sealedSegments.size()
+ ? sealedSegments.get(sealedHead).baseSeq()
+ : previous.baseSeq();
+ manifest.update(headBase, actualBase);
+ }
+ previous.linkSuccessor(spare);
+ sealedSegments.add(previous);
+ active = spare;
}
- active = spare;
hotSpare = null;
// Fresh active just consumed the spare → ask the manager to start
// making the next one immediately, before this segment fills.
@@ -439,13 +917,14 @@ public synchronized void close() {
hotSpare.close();
hotSpare = null;
}
- for (int i = 0, n = sealedSegments.size(); i < n; i++) {
- MmapSegment s = sealedSegments.get(i);
- if (s != null) {
- s.close();
- }
+ for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
+ sealedSegments.get(i).close();
}
sealedSegments.clear();
+ sealedHead = 0;
+ if (manifest != null) {
+ manifest.close();
+ }
}
/**
@@ -461,10 +940,8 @@ public synchronized ObjList drainTrimmable() {
ObjList out = null;
// Sealed segments are in baseSeq order, oldest first; once we hit one
// that isn't fully acked, none of the later ones can be either.
- // Synchronized so the I/O thread's snapshotSealedSegments() can't
- // race against the remove(0) shuffling slots underneath it.
- while (sealedSegments.size() > 0) {
- MmapSegment s = sealedSegments.get(0);
+ while (sealedHead < sealedSegments.size()) {
+ MmapSegment s = sealedSegments.get(sealedHead);
long lastSeq = s.baseSeq() + s.frameCount() - 1;
if (lastSeq > acked) {
break;
@@ -473,7 +950,7 @@ public synchronized ObjList drainTrimmable() {
out = new ObjList<>();
}
out.add(s);
- sealedSegments.remove(0);
+ removeSealedHead();
}
return out;
}
@@ -490,7 +967,7 @@ public synchronized ObjList drainTrimmable() {
* scan cost doesn't matter.
*/
public synchronized MmapSegment findSegmentContaining(long fsn) {
- for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+ for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
MmapSegment s = sealedSegments.get(i);
long base = s.baseSeq();
if (fsn >= base && fsn < base + s.frameCount()) {
@@ -513,7 +990,22 @@ public synchronized MmapSegment findSegmentContaining(long fsn) {
* fallback -- see {@link #nextSealedAfter(MmapSegment)}.
*/
public synchronized MmapSegment firstSealed() {
- return sealedSegments.size() > 0 ? sealedSegments.get(0) : null;
+ return sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null;
+ }
+
+ /**
+ * Returns the oldest fully acknowledged sealed segment without removing
+ * it. The segment manager keeps it owned by the ring until close + unlink
+ * succeeds, so a failed unlink cannot make the path disappear from live
+ * bookkeeping or allow its identifier to be reused.
+ */
+ public synchronized MmapSegment firstTrimmable() {
+ if (sealedHead == sealedSegments.size()) {
+ return null;
+ }
+ MmapSegment segment = sealedSegments.get(sealedHead);
+ long lastSeq = segment.baseSeq() + segment.frameCount() - 1;
+ return lastSeq <= ackedFsn ? segment : null;
}
/** Active segment -- exposed for the I/O thread's "send next batch" path. */
@@ -536,7 +1028,7 @@ public synchronized MmapSegment firstSealed() {
*/
public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) {
long best = -1L;
- for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+ for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
long fsn = sealedSegments.get(i).findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen);
if (fsn > best) {
best = fsn;
@@ -556,7 +1048,8 @@ public MmapSegment getActive() {
* concurrent rotation. Cross-thread readers (typically the I/O loop)
* should use {@link #snapshotSealedSegments(MmapSegment[])} instead.
*/
- public ObjList getSealedSegments() {
+ public synchronized ObjList getSealedSegments() {
+ compactSealedSegments();
return sealedSegments;
}
@@ -600,21 +1093,28 @@ public boolean needsHotSpare() {
* outpaces the I/O thread and sealed segments accumulate well beyond
* any reasonable snapshot-array size.
*
- * Identity match is intentionally avoided: we compare {@code baseSeq}
- * so the loop is robust against the case where {@code current} was
- * trimmed out from under us (already ACK'd before the I/O thread
- * advanced) -- we still return the next segment in baseSeq order rather
- * than failing. Synchronized against rotation.
+ * Each segment publishes its successor once, before rotation exposes that
+ * successor as active. A constant-time head check detects when trimming
+ * removed the immediate successor and falls forward to the oldest live
+ * sealed segment. Synchronized against rotation and head removal.
*/
public synchronized MmapSegment nextSealedAfter(MmapSegment current) {
- long currentBase = current.baseSeq();
- for (int i = 0, n = sealedSegments.size(); i < n; i++) {
- MmapSegment s = sealedSegments.get(i);
- if (s.baseSeq() > currentBase) {
- return s;
- }
+ nextSealedComparisons++;
+ MmapSegment successor = current.successor();
+ if (successor == null) {
+ return null;
}
- return null;
+ if (successor == active) {
+ return null;
+ }
+ MmapSegment first = sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null;
+ if (first != null && successor.baseSeq() >= first.baseSeq()) {
+ return successor;
+ }
+ // Head trimming may have removed the immediate successor while the
+ // I/O cursor still held an older segment. Trims only remove a prefix,
+ // so the current head is the first live segment after that prefix.
+ return first;
}
/**
@@ -634,6 +1134,22 @@ public long publishedFsn() {
return publishedFsn;
}
+ /**
+ * Commits removal of the segment returned by {@link #firstTrimmable()}.
+ * Returns false if concurrent lifecycle activity changed the head.
+ */
+ public synchronized boolean removeTrimmable(MmapSegment segment) {
+ if (sealedHead == sealedSegments.size() || sealedSegments.get(sealedHead) != segment) {
+ return false;
+ }
+ long lastSeq = segment.baseSeq() + segment.frameCount() - 1;
+ if (lastSeq > ackedFsn) {
+ return false;
+ }
+ removeSealedHead();
+ return true;
+ }
+
/**
* Registers a wakeup callback that the producer thread will invoke when
* a hot spare is needed -- either right after a rotation has consumed the
@@ -663,17 +1179,12 @@ public void setManagerWakeup(Runnable wakeup) {
* the I/O loop is about to do.
*/
public synchronized int snapshotSealedSegments(MmapSegment[] target) {
- int n = sealedSegments.size();
- if (n > target.length) {
- for (int i = 0; i < target.length; i++) {
- target[i] = sealedSegments.get(i);
- }
- return -1;
+ int n = sealedSegments.size() - sealedHead;
+ int copyCount = Math.min(n, target.length);
+ for (int i = 0; i < copyCount; i++) {
+ target[i] = sealedSegments.get(sealedHead + i);
}
- for (int i = 0; i < n; i++) {
- target[i] = sealedSegments.get(i);
- }
- return n;
+ return n > target.length ? -1 : n;
}
/**
@@ -689,21 +1200,47 @@ public synchronized long totalSegmentBytes() {
if (a != null) total += a.sizeBytes();
MmapSegment hs = hotSpare;
if (hs != null) total += hs.sizeBytes();
- for (int i = 0, n = sealedSegments.size(); i < n; i++) {
- MmapSegment s = sealedSegments.get(i);
- if (s != null) total += s.sizeBytes();
+ for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
+ total += sealedSegments.get(i).sizeBytes();
}
return total;
}
+ private void compactSealedSegments() {
+ if (sealedHead > 0) {
+ int liveCount = sealedSegments.size() - sealedHead;
+ trimMovedReferences += liveCount;
+ sealedSegments.remove(0, sealedHead - 1);
+ sealedHead = 0;
+ }
+ }
+
+ private void removeSealedHead() {
+ sealedSegments.setQuick(sealedHead++, null);
+ int size = sealedSegments.size();
+ if (sealedHead == size) {
+ sealedSegments.clear();
+ sealedHead = 0;
+ } else if (sealedHead >= 64 && sealedHead >= size - sealedHead) {
+ compactSealedSegments();
+ }
+ }
+
+ /** Returns the sealed-list operation count used by traversal tests. */
+ @TestOnly
+ public static long getNextSealedComparisons() {
+ return nextSealedComparisons;
+ }
+
/**
* Returns the cumulative count of baseSeq comparisons performed by
* {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()}
* (or process start). The count is incremented once per partition pass
* for the median-of-three pivot pick plus once per element compared
- * against the pivot, so a clean run on N segments adds roughly
- * {@code 3 + (hi - lo - 1)} per recursive frame, summing to O(N log N).
- * Exposed for {@code SegmentRingTest} to detect O(N²) regressions
+ * against the pivot ({@code 3 + (hi - lo - 1)} per pass), and by two per
+ * sift-down level when a range falls back to heapsort, so it strictly
+ * upper-bounds the true compare count and sums to O(N log N) on every
+ * input. Exposed for {@code SegmentRingTest} to detect O(N²) regressions
* deterministically.
*/
@TestOnly
@@ -711,22 +1248,71 @@ public static long getSortComparisons() {
return sortComparisons;
}
+ /** Returns the references moved by sealed-list compaction. */
+ @TestOnly
+ public static long getTrimMovedReferences() {
+ return trimMovedReferences;
+ }
+
+ /** Zeroes the counter exposed via {@link #getNextSealedComparisons()}. */
+ @TestOnly
+ public static void resetNextSealedComparisons() {
+ nextSealedComparisons = 0;
+ }
+
/** Zeroes the counter exposed via {@link #getSortComparisons()}. */
@TestOnly
public static void resetSortComparisons() {
sortComparisons = 0;
}
+ /** Zeroes the counter exposed via {@link #getTrimMovedReferences()}. */
+ @TestOnly
+ public static void resetTrimMovedReferences() {
+ trimMovedReferences = 0;
+ }
+
+ /**
+ * Drives the recovery-time baseSeq sort over the whole list. Exposed so
+ * {@code SegmentRingTest} can feed adversarial orders (organ-pipe, mass
+ * duplicates, median-of-three killer, unsigned-boundary keys) straight
+ * into the sort and assert comparison bounds without staging thousands
+ * of segment files on disk.
+ */
+ @TestOnly
+ public static void sortByBaseSeqForTest(ObjList list) {
+ sortByBaseSeq(list, 0, list.size());
+ }
+
/**
- * In-place quicksort over {@code list[lo, hi)} keyed by ascending
- * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²)
- * on already-sorted input that lexicographic readdir produces (our
- * filenames are zero-padded hex of {@code baseSeq}). Recursion depth is
- * bounded by ~2 log₂(N) -- for the documented 16K-segment ceiling, well
- * under the JVM default stack.
+ * In-place introsort over {@code list[lo, hi)} keyed by ascending
+ * unsigned {@code baseSeq}. Median-of-three quicksort handles the readdir
+ * orders a healthy slot produces (lexicographic enumeration of the
+ * generation-numbered filenames yields already-sorted baseSeqs; hashed
+ * directory order is effectively random), and a partition-pass budget of
+ * 2·⌊log₂(N)⌋ demotes any range that keeps splitting badly to in-place
+ * heapsort. Without that budget, Lomuto with a median-of-three pivot is
+ * O(N²) on organ-pipe, duplicate-heavy and median-of-three-killer orders
+ * -- reachable only through corrupted-yet-parseable or operator-copied
+ * headers, but at the documented 16K-segment ceiling that is 10⁷..10⁸
+ * comparisons of startup stall before recovery validation gets to
+ * reject the slot, so the fallback makes O(N log N) unconditional.
+ * Recursion depth stays under log₂(N) (recurse on the smaller side,
+ * loop on the larger), well within the JVM default stack.
*/
private static void sortByBaseSeq(ObjList list, int lo, int hi) {
+ int n = hi - lo;
+ if (n > 1) {
+ sortByBaseSeq(list, lo, hi, 2 * (31 - Integer.numberOfLeadingZeros(n)));
+ }
+ }
+
+ private static void sortByBaseSeq(ObjList list, int lo, int hi, int budget) {
while (hi - lo > 1) {
+ if (budget-- == 0) {
+ heapSortByBaseSeq(list, lo, hi);
+ return;
+ }
int mid = (lo + hi) >>> 1;
long a = list.get(lo).baseSeq();
long b = list.get(mid).baseSeq();
@@ -756,17 +1342,61 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) {
}
swap(list, store, hi - 1);
// Recurse on the smaller partition; loop on the larger to keep
- // recursion depth bounded by log₂(N).
+ // recursion depth bounded by log₂(N). Children inherit the
+ // remaining pass budget: it counts passes along a root-to-leaf
+ // path, so a chain of bad splits exhausts it after ~2 log₂(N)
+ // levels no matter how the work is divided.
if (store - lo < hi - store - 1) {
- sortByBaseSeq(list, lo, store);
+ sortByBaseSeq(list, lo, store, budget);
lo = store + 1;
} else {
- sortByBaseSeq(list, store + 1, hi);
+ sortByBaseSeq(list, store + 1, hi, budget);
hi = store;
}
}
}
+ /**
+ * In-place heapsort over {@code list[lo, hi)} keyed by ascending unsigned
+ * {@code baseSeq}: the introsort fallback for ranges whose partition-pass
+ * budget ran out. Guaranteed O(N log N) for any key distribution and any
+ * initial order; no allocation.
+ */
+ private static void heapSortByBaseSeq(ObjList list, int lo, int hi) {
+ int n = hi - lo;
+ for (int root = (n >>> 1) - 1; root >= 0; root--) {
+ siftDownByBaseSeq(list, lo, root, n);
+ }
+ for (int end = n - 1; end > 0; end--) {
+ swap(list, lo, lo + end);
+ siftDownByBaseSeq(list, lo, 0, end);
+ }
+ }
+
+ private static void siftDownByBaseSeq(ObjList list, int lo, int root, int heapSize) {
+ while (true) {
+ int child = (root << 1) + 1;
+ if (child >= heapSize) {
+ return;
+ }
+ // At most two unsigned compares per level (sibling pick + parent
+ // test); bump the counter by the constant 2 up front -- same
+ // cheap-upper-bound convention as the partition pass.
+ sortComparisons += 2;
+ if (child + 1 < heapSize
+ && Long.compareUnsigned(list.get(lo + child).baseSeq(),
+ list.get(lo + child + 1).baseSeq()) < 0) {
+ child++;
+ }
+ if (Long.compareUnsigned(list.get(lo + root).baseSeq(),
+ list.get(lo + child).baseSeq()) >= 0) {
+ return;
+ }
+ swap(list, lo + root, lo + child);
+ root = child;
+ }
+ }
+
private static void swap(ObjList list, int i, int j) {
if (i == j) return;
MmapSegment tmp = list.get(i);
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java
index 2dec7668..541f7dcf 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java
@@ -27,6 +27,7 @@
import io.questdb.client.SenderConnectionEvent;
import io.questdb.client.SenderConnectionListener;
import io.questdb.client.std.QuietCloseable;
+import org.jetbrains.annotations.TestOnly;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -146,6 +147,11 @@ public long getTotalDelivered() {
return totalDelivered.get();
}
+ @TestOnly
+ public Thread getWorkerThreadForTesting() {
+ return dispatcherThread;
+ }
+
/**
* Non-blocking enqueue. Always admits the new event unless the dispatcher
* is closed or {@code event} is null. Returns {@code true} when the new
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java
index aaa0e4f9..fb796712 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java
@@ -27,6 +27,7 @@
import io.questdb.client.SenderError;
import io.questdb.client.SenderErrorHandler;
import io.questdb.client.std.QuietCloseable;
+import org.jetbrains.annotations.TestOnly;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -209,6 +210,11 @@ public long getTotalDelivered() {
return totalDelivered.get();
}
+ @TestOnly
+ public Thread getWorkerThreadForTesting() {
+ return dispatcherThread;
+ }
+
/**
* True if at least one error has been delivered to a user-installed
* (non-default) handler since this dispatcher started. Used by
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java
index 2c0e90a1..9e01a630 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java
@@ -26,6 +26,7 @@
import io.questdb.client.SenderProgressHandler;
import io.questdb.client.std.QuietCloseable;
+import org.jetbrains.annotations.TestOnly;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -172,6 +173,11 @@ public long getTotalDelivered() {
return totalDelivered.get();
}
+ @TestOnly
+ public Thread getWorkerThreadForTesting() {
+ return dispatcherThread;
+ }
+
/**
* Replace the user-supplied handler. Effective immediately for any
* subsequent delivery. Pass {@code null} to install the no-op default.
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java
new file mode 100644
index 00000000..f09ce94a
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java
@@ -0,0 +1,282 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.std.Crc32c;
+import io.questdb.client.std.FilesFacade;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Crash-safe boundary record for an SF segment chain. Two fixed-size,
+ * independently CRC-protected records alternate on update. Recovery selects
+ * the valid record with the greatest generation, so a torn update cannot
+ * erase the previous committed head/active boundary.
+ */
+final class SfManifest implements QuietCloseable {
+ static final String FILE_NAME = "sf-manifest.bin";
+ private static final Logger LOG = LoggerFactory.getLogger(SfManifest.class);
+ private static final int CRC_OFFSET = 60;
+ private static final long FILE_SIZE = 128;
+ private static final int MAGIC = 0x314d4653; // SFM1 little-endian
+ private static final int RECORD_SIZE = 64;
+ private static final int VERSION = 1;
+ private final int fd;
+ private final FilesFacade filesFacade;
+ private final String path;
+ private long activeBase;
+ private boolean closed;
+ private long generation;
+ private long headBase;
+
+ private SfManifest(FilesFacade filesFacade, String path, int fd,
+ long generation, long headBase, long activeBase) {
+ this.filesFacade = filesFacade;
+ this.path = path;
+ this.fd = fd;
+ this.generation = generation;
+ this.headBase = headBase;
+ this.activeBase = activeBase;
+ }
+
+ static SfManifest create(FilesFacade filesFacade, String dir, long headBase, long activeBase) {
+ String path = dir + "/" + FILE_NAME;
+ int fd = filesFacade.openRWExclusive(path);
+ if (fd < 0) {
+ throw new MmapSegmentException("exclusive create failed for SF manifest " + path);
+ }
+ boolean success = false;
+ try {
+ if (!filesFacade.allocate(fd, FILE_SIZE)) {
+ throw new MmapSegmentException("could not allocate SF manifest " + path);
+ }
+ SfManifest manifest = new SfManifest(filesFacade, path, fd, 0, -1, -1);
+ manifest.update(headBase, activeBase);
+ if (filesFacade.fsyncDir(dir) != 0) {
+ throw new MmapSegmentException("could not sync SF manifest directory " + dir);
+ }
+ success = true;
+ return manifest;
+ } finally {
+ if (!success) {
+ filesFacade.close(fd);
+ filesFacade.remove(path);
+ }
+ }
+ }
+
+ static SfManifest open(FilesFacade filesFacade, String dir) {
+ String path = dir + "/" + FILE_NAME;
+ if (!filesFacade.exists(path)) {
+ return null;
+ }
+ if (filesFacade.length(path) != FILE_SIZE) {
+ // A wrong-sized manifest is creation debris: create() reaches the
+ // full FILE_SIZE via allocate() before writing the first record,
+ // so a mis-sized file proves the crash happened before any
+ // boundary was ever committed — nothing can depend on it yet
+ // (segment flags are stamped only after create() returns). Treat
+ // as absent so startup self-heals; genuine post-creation loss is
+ // still caught by the manifest-required flag check.
+ quarantineDebris(filesFacade, path, "wrong size " + filesFacade.length(path));
+ return null;
+ }
+ int fd = filesFacade.openRW(path);
+ if (fd < 0) {
+ throw new MmapSegmentException("could not open SF manifest " + path);
+ }
+ long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT);
+ try {
+ Record first = readRecord(filesFacade, fd, buffer, 0);
+ Record second = readRecord(filesFacade, fd, buffer, RECORD_SIZE);
+ Record selected;
+ if (first == null) {
+ selected = second;
+ } else if (second == null || first.generation > second.generation) {
+ selected = first;
+ } else {
+ selected = second;
+ }
+ if (selected == null) {
+ // No valid record in either slot. create() makes the first
+ // record durable (write + fsync) before returning, and every
+ // later update() rewrites only ONE slot — so a torn update
+ // leaves the sibling record intact. Zero valid records
+ // therefore proves a creation crash, not boundary loss.
+ // Self-heal by treating the file as absent. If durable state
+ // DID depend on a manifest (flags stamped, i.e. double-slot
+ // bit rot), recovery still fails closed on the
+ // manifest-required flag check.
+ filesFacade.close(fd);
+ quarantineDebris(filesFacade, path, "no valid CRC-protected record");
+ return null;
+ }
+ return new SfManifest(filesFacade, path, fd, selected.generation,
+ selected.headBase, selected.activeBase);
+ } catch (Throwable t) {
+ filesFacade.close(fd);
+ throw t;
+ } finally {
+ Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT);
+ }
+ }
+
+ long activeBase() {
+ return activeBase;
+ }
+
+ @Override
+ public void close() {
+ if (!closed) {
+ closed = true;
+ filesFacade.close(fd);
+ }
+ }
+
+ long headBase() {
+ return headBase;
+ }
+
+ /**
+ * Unlinks {@code dir}'s manifest file. Used when a slot is being reset to
+ * the "nothing durable" state (fresh-start cleanup, close-time drain, or
+ * recovery accepting a segment-less slot as empty). Returns {@code true}
+ * when the file is confirmed gone (removed, or never existed).
+ */
+ static boolean removeFile(FilesFacade filesFacade, String dir) {
+ String path = dir + "/" + FILE_NAME;
+ return filesFacade.remove(path) || !filesFacade.exists(path);
+ }
+
+ synchronized void update(long newHeadBase, long newActiveBase) {
+ if (closed) {
+ throw new IllegalStateException("SF manifest is closed");
+ }
+ // Committed boundaries only ever move forward: head advances on trim,
+ // active advances on rotation. Clamp instead of throwing because the
+ // two writers (producer rotation, manager trim) are serialized on the
+ // ring monitor but may each compute their argument from a snapshot
+ // the other has already moved past — e.g. rotation reads the sealed
+ // list while a trimmed-but-not-yet-removed head segment still sits in
+ // it. Regressing a durable boundary would let a later crash-recovery
+ // demand a segment file the trim path already unlinked (startup would
+ // fail on "missing head segment") or, worse, re-expose stale files
+ // below an already-committed head.
+ if (generation > 0) {
+ if (newHeadBase < headBase) {
+ newHeadBase = headBase;
+ }
+ if (newActiveBase < activeBase) {
+ newActiveBase = activeBase;
+ }
+ }
+ if (newHeadBase < 0 || newActiveBase < newHeadBase) {
+ throw new IllegalArgumentException("invalid SF manifest boundaries");
+ }
+ if (generation > 0 && headBase == newHeadBase && activeBase == newActiveBase) {
+ return;
+ }
+ long nextGeneration = generation + 1;
+ long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT);
+ try {
+ Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0);
+ Unsafe.getUnsafe().putInt(buffer, MAGIC);
+ Unsafe.getUnsafe().putInt(buffer + 4, VERSION);
+ Unsafe.getUnsafe().putLong(buffer + 8, nextGeneration);
+ Unsafe.getUnsafe().putLong(buffer + 16, newHeadBase);
+ Unsafe.getUnsafe().putLong(buffer + 24, newActiveBase);
+ int crc = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET);
+ Unsafe.getUnsafe().putInt(buffer + CRC_OFFSET, crc);
+ long offset = (nextGeneration & 1L) * RECORD_SIZE;
+ if (filesFacade.write(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) {
+ throw new MmapSegmentException("short write updating SF manifest " + path);
+ }
+ if (filesFacade.fsync(fd) != 0) {
+ throw new MmapSegmentException("could not sync SF manifest " + path);
+ }
+ generation = nextGeneration;
+ headBase = newHeadBase;
+ activeBase = newActiveBase;
+ } finally {
+ Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT);
+ }
+ }
+
+ /**
+ * Moves creation-crash debris aside so a subsequent exclusive create can
+ * succeed. Prefers rename (keeps the bytes for postmortem); falls back to
+ * remove; throws when neither works — leaving the debris in place would
+ * wedge every subsequent {@link #create}.
+ */
+ private static void quarantineDebris(FilesFacade filesFacade, String path, String reason) {
+ LOG.warn("SF manifest {} is creation-crash debris ({}); quarantining and starting "
+ + "from the segment files", path, reason);
+ if (filesFacade.rename(path, path + ".corrupt") == 0) {
+ return;
+ }
+ if (filesFacade.remove(path)) {
+ return;
+ }
+ throw new MmapSegmentException("could not quarantine invalid SF manifest " + path);
+ }
+
+ private static Record readRecord(FilesFacade filesFacade, int fd, long buffer, long offset) {
+ Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0);
+ if (filesFacade.read(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) {
+ return null;
+ }
+ if (Unsafe.getUnsafe().getInt(buffer) != MAGIC
+ || Unsafe.getUnsafe().getInt(buffer + 4) != VERSION) {
+ return null;
+ }
+ int expected = Unsafe.getUnsafe().getInt(buffer + CRC_OFFSET);
+ int actual = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET);
+ if (expected != actual) {
+ return null;
+ }
+ long generation = Unsafe.getUnsafe().getLong(buffer + 8);
+ long headBase = Unsafe.getUnsafe().getLong(buffer + 16);
+ long activeBase = Unsafe.getUnsafe().getLong(buffer + 24);
+ if (generation <= 0 || headBase < 0 || activeBase < headBase) {
+ return null;
+ }
+ return new Record(generation, headBase, activeBase);
+ }
+
+ private static final class Record {
+ private final long activeBase;
+ private final long generation;
+ private final long headBase;
+
+ private Record(long generation, long headBase, long activeBase) {
+ this.generation = generation;
+ this.headBase = headBase;
+ this.activeBase = activeBase;
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
index 0b8379de..2bbcc109 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
@@ -36,9 +36,9 @@
* Advisory exclusive lock for a single SF slot directory.
*
* One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx}
- * for the entire lifetime of the engine that owns the slot. The lock is
- * automatically released when the fd is closed — including on hard process
- * exit, since the kernel cleans up file locks for terminated processes.
+ * for the entire lifetime of the engine that owns the slot. Normal teardown
+ * explicitly unlocks it before closing the fd; hard process exit remains a
+ * backstop because the kernel cleans up file locks for terminated processes.
*
* The holder's PID is written to a sibling {@code .lock.pid} file at
* acquisition time. A failed acquisition reads it back so the error message
@@ -116,15 +116,44 @@ public String slotDir() {
return slotDir;
}
- @Override
- public void close() {
- // Closing the fd releases the lock. We do NOT remove the .lock
- // file or the .lock.pid sidecar — a stale PID is harmless (next
- // acquirer overwrites .lock.pid on success).
- if (fd >= 0) {
- Files.close(fd);
+ /**
+ * Explicitly releases the flock and reports whether the release was
+ * confirmed. After a successful unlock the native primitive closes
+ * the descriptor once, best-effort, and this object forgets its numeric
+ * value. It never retries that close: POSIX leaves descriptor state
+ * unspecified after some close failures (notably {@code EINTR}), so a
+ * retry could close an unrelated descriptor that reused the same number.
+ * We do NOT remove the {@code .lock} file or {@code .lock.pid} sidecar; a
+ * stale PID is harmless because the next acquirer overwrites it.
+ *
+ * When the explicit unlock itself fails, the fd is retained so a later
+ * attempt can safely retry the non-consuming unlock operation. Idempotent
+ * once the unlock has succeeded.
+ *
+ * Owners that gate a "slot dir is reusable" signal on the release
+ * (e.g. {@code CursorSendEngine.finishClose} publishing
+ * {@code closeCompleted}) must call this and check the result rather
+ * than {@link #close()}, which is best-effort by contract.
+ *
+ * @return {@code true} if the lock was explicitly released (or was already
+ * released), {@code false} if the OS reported an unlock failure
+ */
+ public synchronized boolean release() {
+ if (fd < 0) {
+ return true;
+ }
+ if (release0(fd) == 0) {
fd = -1;
+ return true;
}
+ return false;
+ }
+
+ @Override
+ public void close() {
+ // QuietCloseable contract: best-effort, no signal. Callers that
+ // must confirm the release use release() and check the result.
+ release();
}
private static String readHolder(String pidPath) {
@@ -152,6 +181,8 @@ private static String readHolder(String pidPath) {
}
}
+ private static native int release0(int fd);
+
private static void writePid(String pidPath) {
long pid;
try {
diff --git a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java
index 259ff10c..1fc86798 100644
--- a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java
+++ b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java
@@ -27,6 +27,8 @@
import io.questdb.client.QueryException;
import io.questdb.client.cutlass.qwp.client.QwpQueryClient;
import org.jetbrains.annotations.TestOnly;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -56,10 +58,19 @@ public final class QueryClientPool implements AutoCloseable {
// close() can never block the caller unbounded. Tunable per pool via
// closeQueryTimeoutMillis(long).
static final long DEFAULT_CLOSE_QUERY_TIMEOUT_MILLIS = 5_000;
+ // Hard cap on close()'s wait for a creation blocked in DNS, TCP, TLS, or
+ // WebSocket setup. A late creator retains the client and native-scratch
+ // cleanup obligation until it returns and observes the closed pool.
+ static final long MAX_CLOSE_CREATION_WAIT_MILLIS = 5_000;
+ private static final Logger LOG = LoggerFactory.getLogger(QueryClientPool.class);
private final long acquireTimeoutMillis;
private final ArrayList all;
private final ArrayDeque available;
private final String configurationString;
+ // Signals completion of internally owned creation lifecycles. Kept
+ // separate from workerReleased so an acquire waiter cannot consume the only
+ // wakeup intended for close().
+ private final Condition creationFinished;
// Test seam. Production connects via QwpQueryClient.connect(); white-box
// tests in io.questdb.client.test.impl reach the package-private constructor
// by reflection to inject a hook that throws a non-RuntimeException
@@ -146,6 +157,7 @@ public QueryClientPool(
this.maxLifetimeMillis = maxLifetimeMillis;
this.all = new ArrayList<>(maxSize);
this.available = new ArrayDeque<>(maxSize);
+ this.creationFinished = lock.newCondition();
this.workerReleased = lock.newCondition();
int built = 0;
// Tracks a worker built by createUnlocked() but not yet added to `all`:
@@ -231,16 +243,11 @@ public QueryWorker acquire() {
// incremented, permanently shrinking pool capacity until
// every acquire() times out. Restoring the reservation
// for any throwable is safe.
- lock.lock();
- inFlightCreations--;
- workerReleased.signal();
- lock.unlock();
// createUnlocked() returns a fully connected client
// (socket + native scratch + I/O thread), so if start()
// threw afterwards we must close it here -- nothing else
- // references it. createUnlocked() already self-cleans
- // when connect() throws, leaving created == null, so
- // this only fires on the start()-throws path.
+ // references it. Keep the creation reservation until
+ // that cleanup finishes so close() cannot return first.
if (created != null) {
try {
created.shutdown();
@@ -249,33 +256,45 @@ public QueryWorker acquire() {
// original creation failure rethrown below.
}
}
+ lock.lock();
+ try {
+ inFlightCreations--;
+ creationFinished.signalAll();
+ workerReleased.signal();
+ } finally {
+ lock.unlock();
+ }
throw new QueryException((byte) 0,
"failed to create query client: " + e.getMessage(), e);
}
lock.lock();
- inFlightCreations--;
if (closed) {
- // Pool was closed mid-creation -- tear the fresh worker
- // down rather than leaking it, but OUTSIDE the lock:
- // shutdown() joins the dispatch thread for up to
- // SHUTDOWN_JOIN_MILLIS, and close()/release()/discard()/
- // cancelIfCurrent() all contend on this lock (whose
- // contract is "held only briefly"). The accounting above
- // already ran under the lock, and the worker never
- // entered `all`, so close()'s snapshot loop cannot race
- // this teardown.
+ // Keep inFlightCreations reserved while shutdown runs
+ // outside the lock. The worker never entered `all`, so
+ // this reservation is close()'s sole ownership record.
lock.unlock();
try {
created.shutdown();
} catch (Throwable ignored) {
// Best-effort: an Error from teardown must not mask
// the closed-pool signal.
+ } finally {
+ lock.lock();
+ try {
+ inFlightCreations--;
+ creationFinished.signalAll();
+ workerReleased.signalAll();
+ } finally {
+ lock.unlock();
+ }
}
throw new QueryException((byte) 0, "QuestDB handle is closed");
}
all.add(created);
// Stamp the first lease id for this freshly built worker.
created.bumpGeneration();
+ inFlightCreations--;
+ creationFinished.signalAll();
return created;
}
if (remainingNanos <= 0) {
@@ -307,6 +326,35 @@ public void close() {
}
closed = true;
workerReleased.signalAll();
+ // A creator can block in DNS, TCP, TLS, WebSocket upgrade, or the
+ // SERVER_INFO handshake. Wait boundedly rather than turning an
+ // unset connect_timeout into an unbounded QuestDB.close(). Timing
+ // out does not abandon the client or its native scratch: the creator
+ // keeps the reservation and, once construction returns, observes
+ // closed and shuts the worker down before releasing ownership.
+ // Preserve interruption while applying the same finite budget.
+ final long creationWaitMillis = Math.max(0,
+ Math.min(acquireTimeoutMillis, MAX_CLOSE_CREATION_WAIT_MILLIS));
+ final long creationWaitNanos = TimeUnit.MILLISECONDS.toNanos(creationWaitMillis);
+ final long creationWaitDeadlineNanos = System.nanoTime() + creationWaitNanos;
+ long creationRemainingNanos = creationWaitNanos;
+ boolean creationWaitInterrupted = false;
+ while (inFlightCreations > 0 && creationRemainingNanos > 0) {
+ try {
+ creationFinished.awaitNanos(creationRemainingNanos);
+ } catch (InterruptedException e) {
+ creationWaitInterrupted = true;
+ }
+ creationRemainingNanos = creationWaitDeadlineNanos - System.nanoTime();
+ }
+ if (creationWaitInterrupted) {
+ Thread.currentThread().interrupt();
+ }
+ if (inFlightCreations > 0) {
+ LOG.warn("QueryClientPool.close(): {} query client creation(s) still in flight after {}ms; "
+ + "each creator retains cleanup ownership until construction returns",
+ inFlightCreations, creationWaitMillis);
+ }
snapshot = new ArrayList<>(all);
} finally {
lock.unlock();
diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java
index e3da539b..3d7be5d9 100644
--- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java
+++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java
@@ -181,8 +181,20 @@ public Sender borrowSender() {
return senderPool.borrow();
}
+ // synchronized so concurrent close() callers serialize THROUGH the bounded
+ // shutdown sequence, not merely through the `closed` flip. `closed` is set
+ // before the teardown chain runs, so a plain volatile guard (or a bare CAS)
+ // would let a second caller observe closed==true and return while the first
+ // is still closing published resources. The monitor makes the losing caller
+ // block until the winner finishes, then it enters, sees `closed` and returns
+ // a no-op. A pool creator that outlives its finite shutdown wait retains and
+ // eventually cleans its unpublished resources on the creator thread; this
+ // monitor does not wait past that budget. No deadlock: the teardown steps
+ // (markClosing/housekeeper.stop()/queryPool.close()/senderPool.close())
+ // never call back into QuestDBImpl.close() on another thread, so nothing
+ // contends for this monitor from within the critical section.
@Override
- public void close() {
+ public synchronized void close() {
if (closed) {
return;
}
diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java
index 1cc214b4..49cac21d 100644
--- a/core/src/main/java/io/questdb/client/impl/SenderPool.java
+++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java
@@ -40,6 +40,7 @@
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Iterator;
+import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
@@ -85,8 +86,8 @@ public final class SenderPool implements AutoCloseable {
// Per-slot wall-clock cap on a single startup-recovery drain. Kept BELOW the
// PoolHousekeeper stop/join budget (PoolHousekeeper.STOP_TIMEOUT_MILLIS) so a
// recovery drain still in flight when close() arrives cannot outlive the
- // housekeeper join -- the residual-budget bound that, together with the
- // early markClosing() signal, keeps close() prompt (C1 fix).
+ // driver join -- the residual-budget bound that, together with the early
+ // markClosing() signal, keeps close() prompt (C1 fix).
//
// This caps only the DRAIN. The recovery build that precedes it is bounded
// separately: recovery delegates force initial_connect_mode=OFF (see
@@ -96,6 +97,15 @@ public final class SenderPool implements AutoCloseable {
// -- the residual window documented on recoverOneSlotStep -- because the
// transport has no application-level connect timeout to clamp it.
private static final long RECOVERY_DRAIN_BUDGET_MILLIS = 1_000;
+ // A direct SenderPool has no PoolHousekeeper to retry a transient startup
+ // recovery failure. Its private driver waits this long between failed
+ // attempts so an unavailable server does not cause a hot retry loop.
+ private static final long RECOVERY_RETRY_INTERVAL_MILLIS = 1_000;
+ // Hard cap on close()'s wait for a creation that is blocked in DNS, TCP,
+ // TLS, or WebSocket setup. The creator retains ownership after this budget:
+ // once construction returns, borrow() observes closed and tears down the
+ // delegate before releasing its reservation (including an SF slot).
+ static final long MAX_CLOSE_CREATION_WAIT_MILLIS = 5_000;
// Hard cap on close()'s outstanding-lease wait. The acquire timeout is a
// BORROW policy -- Long.MAX_VALUE legitimately means "block until a slot
// frees" -- and must never unbound SHUTDOWN: without this cap a forgotten
@@ -106,7 +116,16 @@ public final class SenderPool implements AutoCloseable {
private final long acquireTimeoutMillis;
private final ArrayList all;
private final ArrayDeque available;
+ // Test-only constructor seam. Runs immediately before a possibly-started
+ // recovery driver is joined after constructor failure. Null in production;
+ // lifecycle tests use it to release a deliberately held driver and prove
+ // delegate cleanup happens only after the join.
+ private final Runnable beforeFailedStartupRecoveryJoinHook;
private final String configurationString;
+ // Signals completion of internally owned creation lifecycles. Kept
+ // separate from slotReleased so a capacity waiter cannot consume the only
+ // wakeup intended for close().
+ private final Condition creationFinished;
// User-supplied ingest callbacks, shared across every pooled Sender this
// pool builds. Null -> each sender keeps its loud-not-silent default.
private final SenderConnectionListener connectionListener;
@@ -119,15 +138,19 @@ public final class SenderPool implements AutoCloseable {
// RuntimeException Throwable (e.g. an -ea AssertionError) mid-prewarm,
// exercising the Error-safe delegate cleanup loop.
private final IntFunction senderFactory;
+ // Test seam: runs immediately after a delegate factory returns, before
+ // listener registration and SenderSlot construction. Null in production;
+ // error-safety tests inject a preallocated throwable at this ownership gap.
+ private final Runnable postFactoryHook;
// Factory for startup-recovery delegates. Distinct from senderFactory so a
// recoverer can force a non-blocking initial connect (initial_connect_mode=
- // OFF) regardless of user config: a recovery build runs on the
- // PoolHousekeeper thread and must NOT inherit SYNC (auto-enabled by any
- // reconnect_* knob), which would retry the connect for the whole reconnect
- // budget inside build() -- far past PoolHousekeeper.STOP_TIMEOUT_MILLIS, so
- // a close() landing during that build would make housekeeper.stop()'s join
- // time out and leave the recoverer holding the slot flock after close()
- // returned (M1). Mirrors senderFactory's test seam: an injected factory
+ // OFF) regardless of user config: a recovery build runs on a private direct
+ // driver or the PoolHousekeeper thread and must NOT inherit SYNC
+ // (auto-enabled by any reconnect_* knob), which would retry the connect for
+ // the whole reconnect budget inside build() -- far past
+ // PoolHousekeeper.STOP_TIMEOUT_MILLIS, so a close() landing during that
+ // build could make its driver join time out and leave the recoverer holding
+ // the slot flock after close() returned (M1). Mirrors senderFactory's test seam: an injected factory
// (non-null) drives BOTH paths so white-box recovery tests keep control.
private final IntFunction recoverySenderFactory;
private final ReentrantLock lock = new ReentrantLock();
@@ -148,6 +171,33 @@ public final class SenderPool implements AutoCloseable {
private final Condition slotReleased;
// True iff the configuration enables store-and-forward (sf_dir set).
private final boolean storeAndForward;
+ // Direct pools run one inline recovery attempt for backwards-compatible
+ // startup behavior, then this daemon continues any backlog left by budget
+ // exhaustion or a transient failure. Deferred pools use PoolHousekeeper and
+ // leave this null.
+ private final Object startupRecoverySignal = new Object();
+ private final Thread startupRecoveryThread;
+ // Test-only constructor seam for the failed-retry operation. Production
+ // uses waitForStartupRecoveryRetry(), which performs the one-second signal
+ // wait; lifecycle tests inject an event barrier without wall-clock checks.
+ private final Runnable startupRecoveryWaiter;
+ // Test seam: runs after the direct recovery driver's bounded join returns.
+ // Null in production; lifecycle tests prove the joined thread is quiescent.
+ private volatile Runnable afterStartupRecoveryJoinHook;
+ // Test seam: runs immediately before a capacity-starved borrow enters its
+ // condition wait, while it still holds the pool lock. Null in production;
+ // concurrency tests use a latch here to prove that several borrowers have
+ // all reached the wait path before recovering retired capacity.
+ private volatile Runnable beforeBorrowWaitHook;
+ // Test seam: runs immediately before the direct recovery driver's bounded
+ // join. Null in production; lifecycle tests coordinate close() with a
+ // deliberately held driver.
+ private volatile Runnable beforeStartupRecoveryJoinHook;
+ // Test seam: runs after a capacity-starved borrow's condition wait has
+ // exhausted its positive timeout, before the loop's terminal pass. Null in
+ // production; regression tests release a retired slot here to prove that
+ // the terminal pass re-probes returned capacity before throwing.
+ private volatile Runnable borrowWaitExpiredHook;
// Slots removed from `all` whose delegate is still releasing its flock.
// They keep reserving capacity (and their slotInUse mark) until the
// flock drops, so the cap check and the slot allocator stay consistent
@@ -171,30 +221,52 @@ public final class SenderPool implements AutoCloseable {
// down on another thread. Guarded by lock.
private int pendingLeaseTeardowns;
// Slots whose delegate close() returned with the SF flock still held
- // (the I/O thread refused to stop). Permanently consumed: the index is
+ // because an I/O or manager worker did not stop. Consumed while retired:
// never freed and never reused, so no borrow ever hands out a still-
// locked slot dir. Counted in the cap check so the lost capacity is
- // accounted for. Guarded by lock; only ever ticks for SF slots.
+ // accounted for. NOT necessarily permanent: engine cleanup may be pending
+ // on a worker/I/O-thread exit path, so reprobeRetiredSlots() re-checks
+ // retiredSlots and returns any index whose flock has since dropped.
+ // Guarded by lock; only ever ticks for SF slots.
private int leakedSlots;
+ // Deterministic white-box complexity counter. Counts delegate release
+ // probes performed by direct callbacks and fallback scans. Guarded by lock.
+ private long retiredSlotProbeCount;
+ // The retired slots behind the leakedSlots count: runtime reclaim paths
+ // (discardBroken/reapIdle via reclaimSlot) and the in-range startup-
+ // recovery pass (recoverOneSlotStep, which retains the recoverer slot for
+ // exactly this purpose). Re-probed by reprobeRetiredSlots() so a late
+ // flock release (deferred engine cleanup on a worker exit path) restores
+ // the pool's capacity instead of ratcheting it down until process exit.
+ // Out-of-range startup recoverers are NEVER added: they carry no
+ // leakedSlots tick and their index has no slotInUse entry to free.
+ // Pre-sized to maxSize (every entry keeps a distinct in-range slot index
+ // reserved, so size can never exceed maxSize): add() never grows the
+ // backing array, so a retire (leakedSlots++ then add, under lock) cannot
+ // fail on allocation and strand a counted-but-untracked slot that
+ // reprobeRetiredSlots() could never recover. Guarded by lock.
+ private final ArrayList retiredSlots;
// SF slots currently held by the in-range startup-recovery pass
// (recoverOneSlotStep): each is reserved under `lock` for the
// duration of its drain and counted in the borrow() cap check so a
// concurrent borrow can neither over-allocate past maxSize nor target a
- // dir being recovered. Only ever non-zero on the deferred (housekeeper-
- // driven) recovery path, where recovery overlaps borrow()/return; on the
- // inline construction path the pool is still single-threaded. Guarded by
- // lock; only ever ticks for SF slots.
+ // dir being recovered. The inline constructor drive is single-threaded,
+ // but the continuing direct-pool driver and deferred housekeeper driver can
+ // overlap borrow()/return after publication. Guarded by lock; only ever
+ // ticks for SF slots.
private int recoveringSlots;
// Resumable startup-recovery scan cursor. Advanced only by the single
- // recovery driver -- the inline constructor loop (single-threaded,
- // unpublished) or the PoolHousekeeper thread (the sole deferred driver) --
+ // recovery driver -- the inline constructor loop followed by its private
+ // direct-pool thread, or the PoolHousekeeper thread (the sole deferred
+ // driver) --
// so the cursor itself needs no lock; the per-slot reservation it performs
// (slotInUse/recoveringSlots) is still taken under `lock` because borrow()
// races it. recoveryInRangeNext is the next in-range index in [0, maxSize)
// for pass 1; recoveryOutOfRange / recoveryOutOfRangeNext are the lazily
// built pass-2 work list (same-base slots at index >= maxSize) and its
- // cursor; recoveryComplete latches true when the whole scan finishes or is
- // aborted, making runStartupRecoveryStep()/...ToCompletion() idempotent.
+ // cursor; recoveryComplete latches true only when the whole scan finishes.
+ // A transient build failure or drain timeout leaves the current candidate
+ // pending so a later tick or explicit drive can retry it on the same pool.
private int recoveryInRangeNext;
private IntList recoveryOutOfRange;
private int recoveryOutOfRangeNext;
@@ -209,7 +281,7 @@ public SenderPool(
long maxLifetimeMillis
) {
this(configurationString, minSize, maxSize, acquireTimeoutMillis,
- idleTimeoutMillis, maxLifetimeMillis, null, false, null, null, null);
+ idleTimeoutMillis, maxLifetimeMillis, null, false, null, null, null, null, null, null, null);
}
// Test-only constructor exposing the senderFactory seam: production builds
@@ -238,8 +310,8 @@ public SenderPool(
// reachable-but-not-acking server; the owner (QuestDBImpl) then drives
// recovery one slot per tick on the PoolHousekeeper thread via
// runStartupRecoveryStep(). White-box SF tests call this directly; the
- // in-range recovery pass is concurrency-safe against borrow()/return on the
- // deferred path -- see recoverOneSlotStep().
+ // in-range recovery pass is concurrency-safe against borrow()/return after
+ // pool publication -- see recoverOneSlotStep().
@TestOnly
public SenderPool(
String configurationString,
@@ -253,7 +325,26 @@ public SenderPool(
) {
this(configurationString, minSize, maxSize, acquireTimeoutMillis,
idleTimeoutMillis, maxLifetimeMillis, senderFactory,
- deferStartupRecovery, null, null, null);
+ deferStartupRecovery, null, null, null, null, null, null, null);
+ }
+
+ // Test-only constructor adding a deterministic fault hook for the ownership
+ // gap after a delegate factory returns.
+ @TestOnly
+ public SenderPool(
+ String configurationString,
+ int minSize,
+ int maxSize,
+ long acquireTimeoutMillis,
+ long idleTimeoutMillis,
+ long maxLifetimeMillis,
+ IntFunction senderFactory,
+ boolean deferStartupRecovery,
+ Runnable postFactoryHook
+ ) {
+ this(configurationString, minSize, maxSize, acquireTimeoutMillis,
+ idleTimeoutMillis, maxLifetimeMillis, senderFactory,
+ deferStartupRecovery, null, null, null, postFactoryHook, null, null, null);
}
// Full constructor adding the user-supplied ingest callbacks (error
@@ -273,6 +364,29 @@ public SenderPool(
SenderErrorHandler errorHandler,
SenderConnectionListener connectionListener,
BackgroundDrainerListener drainerListener
+ ) {
+ this(configurationString, minSize, maxSize, acquireTimeoutMillis,
+ idleTimeoutMillis, maxLifetimeMillis, senderFactory,
+ deferStartupRecovery, errorHandler, connectionListener,
+ drainerListener, null, null, null, null);
+ }
+
+ private SenderPool(
+ String configurationString,
+ int minSize,
+ int maxSize,
+ long acquireTimeoutMillis,
+ long idleTimeoutMillis,
+ long maxLifetimeMillis,
+ IntFunction senderFactory,
+ boolean deferStartupRecovery,
+ SenderErrorHandler errorHandler,
+ SenderConnectionListener connectionListener,
+ BackgroundDrainerListener drainerListener,
+ Runnable postFactoryHook,
+ ThreadFactory recoveryThreadFactory,
+ Runnable recoveryWaiter,
+ Runnable beforeFailedRecoveryJoinHook
) {
if (minSize < 0 || maxSize < 1 || minSize > maxSize) {
throw new IllegalArgumentException("invalid pool sizing: min=" + minSize + ", max=" + maxSize);
@@ -291,8 +405,15 @@ public SenderPool(
this.acquireTimeoutMillis = acquireTimeoutMillis;
this.idleTimeoutMillis = idleTimeoutMillis;
this.maxLifetimeMillis = maxLifetimeMillis;
+ this.postFactoryHook = postFactoryHook;
+ this.beforeFailedStartupRecoveryJoinHook = beforeFailedRecoveryJoinHook;
+ this.startupRecoveryWaiter = recoveryWaiter != null
+ ? recoveryWaiter
+ : this::waitForStartupRecoveryRetry;
this.all = new ArrayList<>(maxSize);
this.available = new ArrayDeque<>(maxSize);
+ this.retiredSlots = new ArrayList<>(maxSize);
+ this.creationFinished = lock.newCondition();
this.slotReleased = lock.newCondition();
// Probe the config once, up front: this validates it eagerly (so a
// bad config fails at construction even when minSize == 0) and tells
@@ -324,17 +445,7 @@ public SenderPool(
// propagate without running the cleanup below, leaking every
// already-built delegate's flock + mmap'd ring + I/O thread and
// resurrecting "sf slot already in use" on the next attempt.
- for (int i = 0; i < built; i++) {
- try {
- all.get(i).delegate().close();
- } catch (Throwable ignored) {
- // Best-effort cleanup: a delegate close() can throw an
- // Error (e.g. an -ea AssertionError) as well as a
- // RuntimeException. Swallow either so we still close the
- // remaining pre-warmed slots and rethrow the original
- // construction failure below.
- }
- }
+ closePrewarmedDelegates(built);
throw e;
}
// Prewarm succeeded. Recover any unacked data a previous run left in
@@ -343,18 +454,43 @@ public SenderPool(
// (deferStartupRecovery=true) so QuestDB.build() never blocks on a slow
// or reachable-but-not-acking server; direct constructions run it inline
// here, while still single-threaded and unpublished.
- if (!deferStartupRecovery) {
- runStartupRecoveryToCompletion();
+ Thread recoveryThread = null;
+ try {
+ if (!deferStartupRecovery) {
+ runStartupRecoveryToCompletion();
+ if (storeAndForward && !recoveryComplete) {
+ ThreadFactory threadFactory = recoveryThreadFactory != null
+ ? recoveryThreadFactory
+ : SenderPool::createStartupRecoveryThread;
+ recoveryThread = threadFactory.newThread(this::runStartupRecoveryLoop);
+ recoveryThread.setDaemon(true);
+ }
+ }
+ this.startupRecoveryThread = recoveryThread;
+ if (recoveryThread != null) {
+ recoveryThread.start();
+ }
+ } catch (Throwable e) {
+ // Thread allocation, configuration, or start can throw after
+ // prewarm transferred delegate ownership to this constructor. The
+ // pool cannot be returned, so stop a possibly-started driver before
+ // tearing down delegates it could otherwise still recover against.
+ // Preserve the construction throwable; cleanup is best-effort.
+ markClosing();
+ stopFailedStartupRecoveryDriver(recoveryThread);
+ closePrewarmedDelegates(built);
+ throw e;
}
}
/**
- * Drives startup SF recovery to completion in a single call, bounded by one
- * shared {@code acquireTimeoutMillis} wall-clock budget (and each individual
- * drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the inline
- * construction path -- single-threaded and unpublished -- and by manual /
- * test drives. No-op when SF is off, the pool is shutting down, or recovery
- * has already finished. Idempotent.
+ * Drives startup SF recovery toward completion in a single call, bounded by
+ * one shared {@code acquireTimeoutMillis} wall-clock budget (and each
+ * individual drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the
+ * inline construction path -- single-threaded and unpublished -- and by
+ * manual / test drives. Budget exhaustion or a transient slot failure leaves
+ * the cursor pending for the pool's continuing driver. No-op when SF is off,
+ * the pool is shutting down, or recovery has already finished. Idempotent.
*/
void runStartupRecoveryToCompletion() {
if (!storeAndForward) {
@@ -366,41 +502,73 @@ void runStartupRecoveryToCompletion() {
// accepted for a single borrow, so we reuse it as the total budget; once
// spent, the remaining slots wait for a later attempt (data stays
// durable on disk).
- final long deadlineNanos =
- System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(acquireTimeoutMillis);
+ final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(acquireTimeoutMillis);
+ final long startNanos = System.nanoTime();
while (!closed && !recoveryComplete) {
- long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime());
- if (remainingMillis <= 0) {
+ // Subtract nanoTime samples before comparing them with the duration.
+ // This avoids an overflowed absolute deadline when
+ // acquireTimeoutMillis is Long.MAX_VALUE and keeps the intended
+ // duration comparison explicit.
+ long elapsedNanos = System.nanoTime() - startNanos;
+ if (elapsedNanos >= budgetNanos) {
LOG.warn("startup SF recovery: {}ms budget exhausted; "
- + "skipping remaining slots", acquireTimeoutMillis);
- recoveryComplete = true;
+ + "deferring remaining slots", acquireTimeoutMillis);
return;
}
+ long remainingNanos = budgetNanos - elapsedNanos;
+ long remainingMillis = TimeUnit.NANOSECONDS.toMillis(remainingNanos);
+ if (remainingMillis == 0) {
+ // drain() accepts milliseconds; preserve a positive sub-ms
+ // remainder instead of treating truncation as exhaustion.
+ remainingMillis = 1;
+ }
if (!recoverOneSlotStep(Math.min(remainingMillis, RECOVERY_DRAIN_BUDGET_MILLIS))) {
return;
}
}
}
+ private void runStartupRecoveryLoop() {
+ while (!closed && !recoveryComplete) {
+ boolean hasImmediateWork;
+ try {
+ hasImmediateWork = runStartupRecoveryStep();
+ } catch (Throwable ignored) {
+ // Match PoolHousekeeper: startup recovery is best-effort and a
+ // delegate Error must not permanently kill its only driver.
+ hasImmediateWork = false;
+ }
+ if (closed || recoveryComplete) {
+ return;
+ }
+ if (!hasImmediateWork) {
+ startupRecoveryWaiter.run();
+ if (Thread.currentThread().isInterrupted()) {
+ return;
+ }
+ }
+ }
+ }
+
/**
* Recovers at most ONE stranded managed slot and reports whether more remain.
- * Driven by the {@link PoolHousekeeper} one slot per step, back-to-back, on
- * the reap-loop thread: each step performs at most one drain bounded by
- * {@link #RECOVERY_DRAIN_BUDGET_MILLIS} -- kept below the housekeeper
- * stop/join budget -- on a delegate whose initial connect is forced OFF
+ * The private direct driver or {@link PoolHousekeeper} drives steps
+ * back-to-back: each step performs at most one drain bounded by
+ * {@link #RECOVERY_DRAIN_BUDGET_MILLIS} -- kept below the driver stop/join
+ * budget -- on a delegate whose initial connect is forced OFF
* ({@link #defaultRecoverySender}) so the build makes at most one connect
- * attempt. The housekeeper re-checks {@code stop} between steps while
- * {@link QuestDBImpl#close()} raises the {@code closed} shutdown signal (via
- * {@link #markClosing()}) BEFORE stopping it, so a {@code close()} landing
- * mid-recovery normally only has to wait out a single bounded drain. The one
+ * attempt. Each owner raises the {@code closed} shutdown signal before
+ * joining its driver, so a {@code close()} landing mid-recovery normally
+ * only has to wait out a single bounded drain. The one
* exception is an in-flight connect to a black-holed host, which blocks on
* the OS connect timeout -- see the residual-window note on
* {@link #recoverOneSlotStep}.
* No-op (returns {@code false}) when SF is off, the pool is shutting down, or
* recovery has already finished.
*
- * @return {@code true} if recovery has more work (call again), {@code false}
- * when recovery is complete or the pool is shutting down
+ * @return {@code true} if recovery has more work immediately; {@code false}
+ * when recovery is complete, the pool is shutting down, or a transient
+ * failure deferred the current candidate until a later tick
*/
boolean runStartupRecoveryStep() {
if (!storeAndForward || closed || recoveryComplete) {
@@ -432,8 +600,8 @@ boolean runStartupRecoveryStep() {
*
* The in-range pass reserves each slot index under {@code lock} for the
* duration of its recovery AND counts it in the borrow() capacity check (via
- * {@code recoveringSlots}), so a concurrent borrow on the deferred path can
- * neither target the dir being recovered nor over-allocate past
+ * {@code recoveringSlots}), so a concurrent borrow after pool publication
+ * can neither target the dir being recovered nor over-allocate past
* {@code maxSize}. Prewarmed/borrowed slots (already live, holding their
* flock) are skipped, as are empty slots (a cheap directory probe); only a
* slot that actually holds stranded data spends the step's single drain. The
@@ -441,14 +609,16 @@ boolean runStartupRecoveryStep() {
* {@code slotInUse} entry and are never allocated by borrow().
*
* Best-effort throughout: a build/close Error or a slow drain is logged and
- * never propagates, since the data stays durable on disk for a later attempt;
- * the first build failure or drain timeout latches {@code recoveryComplete}
- * (the failure will very likely repeat for every remaining slot).
+ * never propagates, since the data stays durable on disk for a later attempt.
+ * A build failure or drain timeout stops the current drive but leaves its
+ * candidate pending so a later driver attempt can retry after a transient
+ * condition clears; it does not poison recovery for the life of the pool.
*
- * Boundedness / residual window. Recovery is driven on the
- * PoolHousekeeper thread, and {@code close()} relies on a step finishing
- * within {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. A step is build +
- * drain + close. The drain is capped by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}
+ * Boundedness / residual window. A continuing recovery step runs on
+ * either the direct pool's private driver or the PoolHousekeeper thread, and
+ * {@code close()} relies on a step finishing within
+ * {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. A step is build + drain +
+ * close. The drain is capped by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}
* and the build forces {@code initial_connect_mode=OFF} (see
* {@link #defaultRecoverySender}), so it makes at most ONE connect attempt
* instead of a SYNC reconnect-budget retry loop. That removes the
@@ -456,7 +626,7 @@ boolean runStartupRecoveryStep() {
* One residual window remains and is NOT closed here: a single in-flight
* connect to a black-holed/firewalled host blocks on the OS connect timeout
* (the transport exposes no application-level connect timeout to clamp it).
- * If {@code close()} lands during that one connect the housekeeper join can
+ * If {@code close()} lands during that one connect, its driver join can
* still time out and the detached build releases the slot flock shortly
* after {@code close()} returns. No data is lost (the slot stays durable on
* disk); the exposure is a brief "sf slot already in use" window on an
@@ -470,7 +640,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) {
recoveryComplete = true;
return false;
}
- final boolean[] flockHeld = new boolean[1];
+ final SenderSlot[] retained = new SenderSlot[1];
// Pass 1: in-range managed slots [0, maxSize). Skip live and empty slots
// cheaply; spend the step on the first slot that actually holds data.
@@ -479,8 +649,8 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) {
return false;
}
int i = recoveryInRangeNext;
- // Reserve this index unless prewarm (or a concurrent borrow, on the
- // deferred path) already holds it live. Count the reservation in
+ // Reserve this index unless prewarm (or a concurrent borrow after
+ // publication) already holds it live. Count the reservation in
// recoveringSlots so the borrow() cap check cannot over-allocate
// while this slot is held for recovery.
boolean reserved;
@@ -513,44 +683,49 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) {
recoveryInRangeNext++;
continue;
}
- // A real candidate -> spend the step on it. Advance the cursor first
- // so a resume never reprocesses this index.
- recoveryInRangeNext++;
- boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, flockHeld);
+ // A real candidate -> spend the step on it. Advance the cursor only
+ // after success so a transient build/drain failure remains retryable.
+ boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, retained);
lock.lock();
try {
// Release the recovery reservation accounting; from here either
// leakedSlots (retire) or the freed index carries the cap math.
recoveringSlots--;
- if (flockHeld[0]) {
- // close() bailed early with the I/O thread still running and
- // the flock still held. Retire the slot permanently (mirror
+ if (retained[0] != null) {
+ // close() retained the flock because an I/O or manager
+ // worker did not stop. Retire the slot (mirror
// discardBroken/reapIdle): keep slotInUse[i] set and count it
// in leakedSlots so the borrow() cap math accounts for the
// lost capacity and no later borrow ever reuses the
- // still-locked dir.
+ // still-locked dir. Keep the recoverer in retiredSlots so
+ // reprobeRetiredSlots() restores the capacity once the
+ // deferred engine cleanup releases the flock — without it
+ // the retirement would be permanent even after the release
+ // (fatal at maxSize=1: every later borrow would time out).
leakedSlots++;
- LOG.warn("startup SF recovery: slot {} retired permanently: delegate close() returned with "
- + "the flock still held (I/O thread refused to stop); pool capacity reduced by 1, "
- + "now {} of {} usable [leakedSlots={}]",
+ addRetiredSlot(retained[0]);
+ LOG.warn("startup SF recovery: slot {} retired: delegate close() returned with "
+ + "the flock still held (I/O or manager worker did not stop); pool capacity reduced by 1, "
+ + "now {} of {} usable [leakedSlots={}]; the slot is re-probed and recovered "
+ + "if the worker releases the flock later",
i, maxSize - leakedSlots, maxSize, leakedSlots);
} else {
slotInUse[i] = false;
- // On the deferred path a borrow may be waiting on capacity
- // this recovery held; the freed index can now admit a
- // creation.
+ // On a post-publication drive, a borrow may be waiting on
+ // capacity this recovery held; the freed index can now admit
+ // a creation.
slotReleased.signal();
}
} finally {
lock.unlock();
}
if (stopScan) {
- // A build failure or drain timeout that will very likely repeat
- // for every remaining slot -- abort the scan; the data stays
- // durable on disk for a later attempt. Do not start pass 2.
- recoveryComplete = true;
+ // Stop this drive without advancing the cursor. The same live
+ // pool retries this candidate after the transient condition is
+ // removed instead of requiring pool recreation.
return false;
}
+ recoveryInRangeNext++;
return true;
}
@@ -571,27 +746,34 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) {
if (closed) {
return false;
}
- int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext++);
+ int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext);
String slotPath = sfDir + "/" + slotBaseId + "-" + idx;
if (!OrphanScanner.isCandidateOrphan(slotPath)) {
+ recoveryOutOfRangeNext++;
continue;
}
- boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, flockHeld);
- if (flockHeld[0]) {
+ boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, retained);
+ if (retained[0] != null) {
// Out of the pool's [0, maxSize) capacity range: there is no
// slotInUse entry to retire and no future borrow targets this
- // dir, so a still-held flock only leaks this recoverer's I/O
- // thread (a best-effort teardown loss, logged). Crucially we do
+ // dir, so a still-held flock only leaks this recoverer's
+ // worker-reachable resources (a best-effort teardown loss,
+ // logged). Crucially we do
// NOT touch leakedSlots -- that would wrongly shrink the
- // in-range pool capacity.
+ // in-range pool capacity -- and we do NOT add to retiredSlots:
+ // there is no capacity to recover, and freeSlotIndex(idx)
+ // would index past the slotInUse array (sized maxSize).
LOG.warn("startup SF recovery: out-of-range slot {} closed with the flock still held "
- + "(I/O thread refused to stop); its data is durable on disk for a later attempt",
+ + "(I/O or manager worker did not stop); its data is durable on disk for a later attempt",
slotPath);
}
if (stopScan) {
- recoveryComplete = true;
+ // Keep the out-of-range cursor on this candidate. In particular,
+ // a transient flock/build collision must be retried after the
+ // primary sender returns; no capacity bookkeeping is involved.
return false;
}
+ recoveryOutOfRangeNext++;
return true;
}
@@ -599,27 +781,46 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) {
return false;
}
+ private void closePrewarmedDelegates(int built) {
+ for (int i = 0; i < built; i++) {
+ try {
+ all.get(i).delegate().close();
+ } catch (Throwable ignored) {
+ // Best-effort cleanup: one delegate close failure must not
+ // strand later prewarmed delegates or replace the original
+ // construction throwable.
+ }
+ }
+ }
+
+ private static Thread createStartupRecoveryThread(Runnable runnable) {
+ return new Thread(runnable, "questdb-sender-pool-recovery");
+ }
+
/**
* Drains one candidate orphan slot dir within {@code remainingMillis},
* best-effort and never throwing. Builds a recoverer on {@code slotIndex}
* (whose {@link #defaultSender} derives the dir {@code -slotIndex}),
* drains its unacked data, and closes the delegate. Shared by both recovery
* passes -- the in-range pass and the out-of-range pass -- which differ only
- * in their slot bookkeeping, handled by the caller via {@code flockHeld}.
+ * in their slot bookkeeping, handled by the caller via {@code retainedOut}.
*
- * @param flockHeld single-element out-param set to {@code true} iff a
- * recoverer was built and its {@code close()} returned with
- * the flock still held (the I/O thread refused to stop)
+ * @param retainedOut single-element out-param set to the recoverer iff one
+ * was built and its {@code close()} returned with the
+ * flock still held because a worker did not stop; the
+ * in-range caller keeps it in {@link #retiredSlots} so a
+ * late flock release can be re-probed. {@code null} when
+ * the flock was released (or no recoverer was built).
* @return {@code true} if a build/drain failure occurred that will very
* likely repeat for every remaining slot, so the caller should stop scanning
*/
private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath,
- long remainingMillis, boolean[] flockHeld) {
- flockHeld[0] = false;
+ long remainingMillis, SenderSlot[] retainedOut) {
+ retainedOut[0] = null;
// Hoisted so the flock check after the try can consult it:
// createRecoverer() takes the slot flock on -slotIndex, and
- // delegate().close() can early-return with the I/O thread still running
- // (flock still held).
+ // delegate().close() can retain it when an I/O or manager worker does
+ // not stop.
SenderSlot recoverer = null;
boolean stopScan = false;
try {
@@ -630,15 +831,15 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath,
// Recovery delegate: forced OFF-mode initial connect (see
// createRecoverer / defaultRecoverySender), so this build does
// at most ONE connect attempt -- it never inherits the SYNC
- // reconnect-budget retry loop that would block this
- // (PoolHousekeeper) thread for minutes (M1).
+ // reconnect-budget retry loop that would block this recovery
+ // driver for minutes (M1).
recoverer = createRecoverer(slotIndex);
} catch (Throwable buildErr) {
// A build/connect failure (e.g. server unreachable) will very
// likely repeat for every remaining slot, so stop here rather
// than pay a connect timeout per slot.
LOG.warn("startup SF recovery: could not open slot {} ({}); "
- + "skipping remaining slots", slotPath, buildErr.toString());
+ + "deferring this and remaining slots", slotPath, buildErr.toString());
return true;
}
try {
@@ -648,13 +849,14 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath,
// same reasoning as the build-failure case above.
if (!recoverer.delegate().drain(remainingMillis)) {
LOG.warn("startup SF recovery: drain did not ack slot {} "
- + "within {}ms; skipping remaining slots",
+ + "within {}ms; deferring this and remaining slots",
slotPath, remainingMillis);
stopScan = true;
}
} catch (Throwable drainErr) {
- LOG.warn("startup SF recovery: drain failed for slot {} ({})",
+ LOG.warn("startup SF recovery: drain failed for slot {} ({}); deferring it",
slotPath, drainErr.toString());
+ stopScan = true;
} finally {
try {
recoverer.delegate().close();
@@ -664,11 +866,12 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath,
}
}
} catch (Throwable scanErr) {
- LOG.warn("startup SF recovery: scan failed for slot {} ({})",
+ LOG.warn("startup SF recovery: scan failed for slot {} ({}); deferring it",
slotPath, scanErr.toString());
+ stopScan = true;
}
- if (recoverer != null) {
- flockHeld[0] = !flockReleased(recoverer);
+ if (recoverer != null && !flockReleased(recoverer)) {
+ retainedOut[0] = recoverer;
}
return stopScan;
}
@@ -715,33 +918,26 @@ public PooledSender borrow() {
// idempotent, so undoing the reservation for any
// throwable is safe.
lock.lock();
- inFlightCreations--;
- freeSlotIndex(slotIndex);
- slotReleased.signal();
- lock.unlock();
+ try {
+ inFlightCreations--;
+ freeSlotIndex(slotIndex);
+ creationFinished.signalAll();
+ slotReleased.signal();
+ } finally {
+ lock.unlock();
+ }
throw e;
}
lock.lock();
- inFlightCreations--;
if (closed) {
- // Pool was closed mid-creation -- destroy the new connection
- // rather than leaking it. Other waiters have been signaled
- // by close() already. The delegate is closed OUTSIDE the
- // lock (mirroring retireLease): its close() can block for
- // seconds (bounded ack drain, drainer-pool wind-down) or
- // longer (unbounded I/O-thread latch await behind an
- // OS-level connect), which held here would stall close(),
- // giveBack/retireLease and reapIdle behind the pool lock.
- // Accounting first, under the lock: for an SF slot the
- // index reservation moves from inFlightCreations to
- // closingSlots until the close below releases the flock,
- // and pendingLeaseTeardowns keeps the out-of-lock close
- // visible to close()'s outstanding-teardown wait.
+ // Pool was closed mid-creation. Keep inFlightCreations
+ // reserved while the delegate is closed outside the
+ // lock: close() waits on that counter, so the creation
+ // remains internally owned until its teardown completes.
boolean reserved = created.slotIndex() >= 0;
if (reserved) {
closingSlots++;
}
- pendingLeaseTeardowns++;
lock.unlock();
try {
created.delegate().close();
@@ -749,31 +945,56 @@ public PooledSender borrow() {
// Best-effort: an Error (e.g. -ea AssertionError)
// from teardown must not mask the closed-pool signal.
} finally {
- // Re-lock to reclaim the SF slot index and signal a
- // close() waiting on this teardown. MUST run even if
- // the delegate close threw, otherwise the slot stays
- // reserved forever and close() waits out its full
- // budget on a teardown that already happened.
lock.lock();
- pendingLeaseTeardowns--;
- if (reserved) {
- reclaimSlot(created, " after closed-mid-creation teardown");
+ try {
+ if (reserved) {
+ reclaimSlot(created, " after closed-mid-creation teardown");
+ }
+ inFlightCreations--;
+ creationFinished.signalAll();
+ slotReleased.signalAll();
+ } finally {
+ lock.unlock();
}
- slotReleased.signalAll();
- lock.unlock();
}
throw new LineSenderException("QuestDB handle is closed");
}
all.add(created);
created.bumpGeneration();
+ inFlightCreations--;
+ creationFinished.signalAll();
return new PooledSender(created, created.generation());
}
+ // Capacity-starved: re-probe retired slots BEFORE the terminal
+ // timeout check — a deferred engine cleanup may have released a
+ // flock since the retire, and the freed index can admit a
+ // creation right now. The delegate normally signals this pool
+ // after deferred release, while this probe also covers delegates
+ // that do not expose that notification and release/listener races.
+ // Ordering matters twice over: a
+ // zero-timeout (try-once) borrow must get its one probe before
+ // throwing, and a borrower whose awaitNanos budget just expired
+ // must get a final probe on its wake-up pass instead of timing
+ // out on capacity that has already come back.
+ if (reprobeRetiredSlots()) {
+ continue;
+ }
if (remainingNanos <= 0) {
throw new LineSenderException(
"timed out waiting for a Sender from the pool after " + acquireTimeoutMillis + "ms");
}
try {
+ Runnable beforeWaitHook = beforeBorrowWaitHook;
+ if (beforeWaitHook != null) {
+ beforeWaitHook.run();
+ }
remainingNanos = slotReleased.awaitNanos(remainingNanos);
+ if (remainingNanos <= 0) {
+ Runnable hook = borrowWaitExpiredHook;
+ if (hook != null) {
+ hook.run();
+ }
+ }
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new LineSenderException("interrupted while waiting for a Sender from the pool");
@@ -786,12 +1007,22 @@ public PooledSender borrow() {
}
}
+ @TestOnly
+ public void setBeforeBorrowWaitHook(Runnable hook) {
+ this.beforeBorrowWaitHook = hook;
+ }
+
+ @TestOnly
+ public void setBorrowWaitExpiredHook(Runnable hook) {
+ this.borrowWaitExpiredHook = hook;
+ }
+
/**
* Raises the shutdown signal early -- without tearing down live delegates --
- * so an in-flight startup-recovery step driven on the {@link PoolHousekeeper}
- * thread stops promptly between slots. {@link QuestDBImpl#close()} calls this
- * BEFORE stopping the housekeeper, so the housekeeper join cannot time out
- * waiting on a fresh slot's recovery drain. Idle-delegate teardown and the
+ * so an in-flight startup-recovery step stops promptly between slots. Direct
+ * {@link #close()} calls this before joining its private driver;
+ * {@link QuestDBImpl#close()} calls it before stopping the housekeeper.
+ * Idle-delegate teardown and the
* outstanding-lease wait still happen in {@link #close()} (guarded by
* {@code closeStarted}, so this early signal never short-circuits them);
* borrowed delegates returned after this signal are torn down on the
@@ -800,6 +1031,9 @@ public PooledSender borrow() {
*/
void markClosing() {
closed = true;
+ synchronized (startupRecoverySignal) {
+ startupRecoverySignal.notifyAll();
+ }
}
/**
@@ -809,6 +1043,9 @@ void markClosing() {
* free their native memory under its feet -- a use-after-free / SEGV, not
* an exception (C1). Instead:
*
+ *
waits boundedly for internally owned creations; a late creator keeps
+ * ownership and performs its closed-mid-creation teardown asynchronously;
+ * then
*
waits boundedly (up to {@code acquireTimeoutMillis}, hard-capped at
* {@link #MAX_CLOSE_LEASE_WAIT_MILLIS}) for outstanding leases to come home -- {@link #giveBack} and {@link #discardBroken}
* observe {@code closed} and tear each delegate down on the returning
@@ -823,6 +1060,11 @@ void markClosing() {
*/
@Override
public void close() {
+ // Direct pools own their continuing recovery driver. Stop it before
+ // snapshotting/closing delegates; deferred pools have a null thread and
+ // QuestDBImpl stops their external PoolHousekeeper before calling here.
+ markClosing();
+ stopStartupRecoveryDriver();
SenderSlot[] idleSnapshot;
lock.lock();
try {
@@ -835,6 +1077,35 @@ public void close() {
closed = true;
// Wake parked borrowers so they observe the shutdown and throw.
slotReleased.signalAll();
+ // A creator can block in DNS, TCP, TLS, or WebSocket setup. Wait
+ // boundedly rather than turning an unset connect_timeout into an
+ // unbounded QuestDB.close(). Timing out does not abandon ownership:
+ // the reservation and any SF slot stay assigned to the creator,
+ // which observes closed and tears down a late result before releasing
+ // them. Preserve the caller's interrupt status while still applying
+ // the same finite wait budget.
+ final long creationWaitMillis = Math.max(0,
+ Math.min(acquireTimeoutMillis, MAX_CLOSE_CREATION_WAIT_MILLIS));
+ final long creationWaitNanos = TimeUnit.MILLISECONDS.toNanos(creationWaitMillis);
+ final long creationWaitDeadlineNanos = System.nanoTime() + creationWaitNanos;
+ long creationRemainingNanos = creationWaitNanos;
+ boolean creationWaitInterrupted = false;
+ while (inFlightCreations > 0 && creationRemainingNanos > 0) {
+ try {
+ creationFinished.awaitNanos(creationRemainingNanos);
+ } catch (InterruptedException e) {
+ creationWaitInterrupted = true;
+ }
+ creationRemainingNanos = creationWaitDeadlineNanos - System.nanoTime();
+ }
+ if (creationWaitInterrupted) {
+ Thread.currentThread().interrupt();
+ }
+ if (inFlightCreations > 0) {
+ LOG.warn("SenderPool.close(): {} sender creation(s) still in flight after {}ms; "
+ + "each creator retains cleanup ownership and releases its SF slot when construction returns",
+ inFlightCreations, creationWaitMillis);
+ }
// Bounded graceful wait for outstanding leases. A slot is borrowed
// iff it is in `all` but not in `available`; retireLease's
// delegate-close section (running outside the lock on a returning
@@ -934,6 +1205,62 @@ public void giveBack(PooledSender ps) {
retireLease(ps, " during pool shutdown");
}
+ @TestOnly
+ private void setStartupRecoveryJoinHooks(Runnable beforeJoinHook, Runnable afterJoinHook) {
+ this.beforeStartupRecoveryJoinHook = beforeJoinHook;
+ this.afterStartupRecoveryJoinHook = afterJoinHook;
+ }
+
+ private void stopFailedStartupRecoveryDriver(Thread recoveryThread) {
+ if (recoveryThread == null || recoveryThread == Thread.currentThread()) {
+ return;
+ }
+ if (beforeFailedStartupRecoveryJoinHook != null) {
+ beforeFailedStartupRecoveryJoinHook.run();
+ }
+ boolean interrupted = false;
+ while (recoveryThread.isAlive()) {
+ try {
+ recoveryThread.join();
+ } catch (InterruptedException e) {
+ interrupted = true;
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void stopStartupRecoveryDriver() {
+ if (startupRecoveryThread == null || startupRecoveryThread == Thread.currentThread()) {
+ return;
+ }
+ if (beforeStartupRecoveryJoinHook != null) {
+ beforeStartupRecoveryJoinHook.run();
+ }
+ try {
+ startupRecoveryThread.join(PoolHousekeeper.STOP_TIMEOUT_MILLIS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ if (afterStartupRecoveryJoinHook != null) {
+ afterStartupRecoveryJoinHook.run();
+ }
+ }
+
+ private void waitForStartupRecoveryRetry() {
+ synchronized (startupRecoverySignal) {
+ if (closed || recoveryComplete) {
+ return;
+ }
+ try {
+ startupRecoverySignal.wait(RECOVERY_RETRY_INTERVAL_MILLIS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
/**
* Retires one lease on the calling (borrower's) thread: validates the
* lease generation under the lock, removes the slot from {@code all},
@@ -1002,7 +1329,9 @@ private void retireLease(PooledSender ps, String context) {
pendingLeaseTeardowns--;
if (reserved) {
// Free the index only when the flock was released; a slot
- // left locked is retired permanently.
+ // left locked is retired into retiredSlots, recoverable
+ // by reprobeRetiredSlots() if the deferred cleanup drops
+ // the flock later.
reclaimSlot(s, context);
}
slotReleased.signalAll();
@@ -1028,6 +1357,11 @@ public void reapIdle() {
if (closed) {
return;
}
+ // Housekeeper tick doubles as the retired-slot recovery driver:
+ // a slot retired because its worker did not stop is re-probed
+ // here and returns to the free set once the deferred cleanup
+ // finally released its flock.
+ reprobeRetiredSlots();
Iterator it = available.iterator();
while (it.hasNext() && all.size() > minSize) {
SenderSlot s = it.next();
@@ -1067,7 +1401,7 @@ public void reapIdle() {
}
// Return reserved SF slot indices to the free set -- but only for
// slots whose delegate confirmed the flock was released. A slot
- // left locked (I/O thread refused to stop) is retired permanently.
+ // left locked because a worker did not stop is retired permanently.
if (storeAndForward) {
lock.lock();
try {
@@ -1106,12 +1440,15 @@ public int totalSize() {
}
/**
- * Snapshot of the number of SF slots permanently retired because a
- * delegate {@code close()} returned with the slot flock still held (the
- * I/O thread refused to stop). Each leaked slot permanently lowers the
- * pool's effective capacity ({@code maxSize - leakedSlotCount()}). A
- * non-zero, growing value explains a pool that has started timing out
- * every {@code borrow()}. For metrics and tests.
+ * Snapshot of the number of SF slots currently retired because a
+ * delegate {@code close()} returned with the slot flock still held after
+ * an I/O or manager worker did not stop. Each leaked slot lowers the
+ * pool's effective capacity ({@code maxSize - leakedSlotCount()}) while
+ * retired. Retired slots are re-probed (housekeeper tick and
+ * capacity-starved borrows) and recovered once the delegate's deferred
+ * cleanup releases the flock, so the count can go back down; a non-zero,
+ * persistent value means a worker is still wedged and explains a pool
+ * that has started timing out {@code borrow()}. For metrics and tests.
*/
public int leakedSlotCount() {
lock.lock();
@@ -1122,8 +1459,39 @@ public int leakedSlotCount() {
}
}
+ private SenderSlot createSlot(IntFunction factory, int slotIndex) {
+ Sender delegate = factory.apply(slotIndex);
+ try {
+ if (postFactoryHook != null) {
+ postFactoryHook.run();
+ }
+ SenderSlot slot = new SenderSlot(delegate, this, slotIndex);
+ if (delegate instanceof QwpWebSocketSender) {
+ ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(
+ () -> recoverReleasedSlot(slot));
+ }
+ return slot;
+ } catch (Throwable failure) {
+ if (delegate instanceof QwpWebSocketSender) {
+ try {
+ ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(null);
+ } catch (Throwable deregistrationFailure) {
+ addSuppressed(failure, deregistrationFailure);
+ }
+ }
+ if (delegate != null) {
+ try {
+ delegate.close();
+ } catch (Throwable closeFailure) {
+ addSuppressed(failure, closeFailure);
+ }
+ }
+ throw failure;
+ }
+ }
+
private SenderSlot createUnlocked(int slotIndex) {
- return new SenderSlot(senderFactory.apply(slotIndex), this, slotIndex);
+ return createSlot(senderFactory, slotIndex);
}
/**
@@ -1134,7 +1502,18 @@ private SenderSlot createUnlocked(int slotIndex) {
* {@link #drainCandidateSlotForRecovery}.
*/
private SenderSlot createRecoverer(int slotIndex) {
- return new SenderSlot(recoverySenderFactory.apply(slotIndex), this, slotIndex);
+ return createSlot(recoverySenderFactory, slotIndex);
+ }
+
+ private static void addSuppressed(Throwable failure, Throwable cleanupFailure) {
+ if (failure != cleanupFailure) {
+ try {
+ failure.addSuppressed(cleanupFailure);
+ } catch (Throwable ignored) {
+ // Preserve the original construction failure even if recording
+ // the secondary cleanup failure cannot allocate.
+ }
+ }
}
private Sender defaultSender(int slotIndex) {
@@ -1144,9 +1523,10 @@ private Sender defaultSender(int slotIndex) {
/**
* Same managed-slot delegate as {@link #defaultSender}, but with the
* initial connect forced to {@link Sender.InitialConnectMode#OFF}. Used
- * only for startup recovery, which runs on the PoolHousekeeper thread: OFF
- * makes the build do at most ONE connect attempt instead of retrying for
- * the whole reconnect budget (SYNC, auto-enabled by any reconnect_* knob),
+ * only for startup recovery, which runs on a private direct driver or the
+ * PoolHousekeeper thread: OFF makes the build do at most ONE connect attempt
+ * instead of retrying for the whole reconnect budget (SYNC, auto-enabled by
+ * any reconnect_* knob),
* keeping a recovery step bounded below
* {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. See M1 / the residual-window
* note on {@link #recoverOneSlotStep}.
@@ -1215,8 +1595,8 @@ private Sender buildManagedSlotSender(int slotIndex, boolean forRecovery) {
// its OWN slot (the one recoverOneSlotStep is processing); it must
// never start a BackgroundDrainerPool for sibling/foreign orphans.
// If it did, the delegate's close() -- called from
- // drainCandidateSlotForRecovery() on the PoolHousekeeper thread,
- // BEFORE its cursorEngine.close() releases the slot flock -- would
+ // drainCandidateSlotForRecovery() on the recovery driver, BEFORE
+ // its cursorEngine.close() releases the slot flock -- would
// block in BackgroundDrainerPool.close() for up to
// GRACEFUL_DRAIN_MILLIS + STOP_GRACE_MILLIS (3s) against a
// reachable-but-not-acking server. That overruns a recovery step's
@@ -1271,7 +1651,7 @@ private void freeSlotIndex(int idx) {
* non-QWP delegate never holds an SF flock, so it is always treated as
* released. A {@link QwpWebSocketSender} reports it via
* {@link QwpWebSocketSender#isSlotLockReleased()} -- false means close()
- * bailed early with the I/O thread still running and the flock still held.
+ * retained the flock because an I/O or manager worker did not stop.
*/
private static boolean flockReleased(SenderSlot s) {
Sender d = s.delegate();
@@ -1281,11 +1661,14 @@ private static boolean flockReleased(SenderSlot s) {
/**
* Reclaims one SF slot after its delegate's {@code close()} has been
* attempted. When the flock was released the index returns to the free
- * set; when {@code close()} returned with the flock still held (the I/O
- * thread refused to stop) the slot is retired permanently --
- * {@code leakedSlots++} and {@code slotInUse[idx]} stays set -- so the cap
- * math accounts for the lost capacity and no later borrow ever reuses the
- * still-locked dir. Either way {@code closingSlots} is decremented.
+ * set; when {@code close()} returned with the flock still held because an
+ * I/O or manager worker did not stop, the slot is retired --
+ * {@code leakedSlots++}, {@code slotInUse[idx]} stays set, and the sender
+ * joins {@code retiredSlots} -- so the cap math accounts for the lost
+ * capacity and no borrow reuses the still-locked dir unless
+ * {@link #reprobeRetiredSlots} later observes the deferred cleanup's
+ * release and recovers the index. Either way {@code closingSlots} is
+ * decremented.
*
* Caller must hold {@code lock} and is responsible for signalling waiters
* (only the free path admits a new creation). Shared by
@@ -1303,9 +1686,87 @@ private boolean reclaimSlot(SenderSlot s, String context) {
return true;
}
leakedSlots++;
- LOG.warn("SF slot {} retired permanently{}: delegate close() returned with the flock still held " +
- "(I/O thread refused to stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]",
+ addRetiredSlot(s);
+ LOG.warn("SF slot {} retired{}: delegate close() returned with the flock still held " +
+ "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable " +
+ "[leakedSlots={}]; the slot is re-probed and recovered if the worker releases the flock later",
s.slotIndex(), context, maxSize - leakedSlots, maxSize, leakedSlots);
return false;
}
+
+ private boolean recoverReleasedSlot(SenderSlot s) {
+ lock.lock();
+ try {
+ int retiredIndex = s.retiredIndex();
+ if (retiredIndex < 0
+ || retiredIndex >= retiredSlots.size()
+ || retiredSlots.get(retiredIndex) != s) {
+ // The callback raced retirement, or is stale/duplicate. The
+ // retirement path probes before insertion, and periodic scans
+ // remain as a fallback, so no capacity can be lost here.
+ return false;
+ }
+ retiredSlotProbeCount++;
+ if (!flockReleased(s)) {
+ return false;
+ }
+ recoverRetiredSlotAt(retiredIndex);
+ slotReleased.signalAll();
+ return true;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Re-probes every retired slot (see {@link #reclaimSlot}) and returns to
+ * the free set any whose delegate now reports the flock released — the
+ * deferred engine cleanup (manager-worker or I/O-thread exit path) has
+ * run since the retire. Restores {@code leakedSlots} capacity and signals
+ * waiters so a parked borrow can admit a creation immediately.
+ *
+ * Caller must hold {@code lock}. The probe is cheap on the delegate side
+ * ({@code isSlotLockReleased()} reads volatiles, and only re-arms the
+ * shared flock-release retry in the rare orphaned-retry state), so
+ * holding the pool lock across it cannot stall behind delegate teardown.
+ *
+ * @return {@code true} if at least one slot's capacity was recovered
+ */
+ private boolean reprobeRetiredSlots() {
+ boolean recovered = false;
+ for (int i = retiredSlots.size() - 1; i >= 0; i--) {
+ SenderSlot s = retiredSlots.get(i);
+ retiredSlotProbeCount++;
+ if (flockReleased(s)) {
+ recoverRetiredSlotAt(i);
+ recovered = true;
+ }
+ }
+ if (recovered) {
+ slotReleased.signalAll();
+ }
+ return recovered;
+ }
+
+ private void addRetiredSlot(SenderSlot s) {
+ s.retiredIndex(retiredSlots.size());
+ retiredSlots.add(s);
+ }
+
+ private void recoverRetiredSlotAt(int retiredIndex) {
+ SenderSlot s = retiredSlots.get(retiredIndex);
+ int last = retiredSlots.size() - 1;
+ if (retiredIndex < last) {
+ SenderSlot moved = retiredSlots.get(last);
+ retiredSlots.set(retiredIndex, moved);
+ moved.retiredIndex(retiredIndex);
+ }
+ retiredSlots.remove(last);
+ s.retiredIndex(-1);
+ leakedSlots--;
+ freeSlotIndex(s.slotIndex());
+ LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " +
+ "pool capacity restored, now {} of {} usable [leakedSlots={}]",
+ s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots);
+ }
}
diff --git a/core/src/main/java/io/questdb/client/impl/SenderSlot.java b/core/src/main/java/io/questdb/client/impl/SenderSlot.java
index 19c93671..5d9ed6ff 100644
--- a/core/src/main/java/io/questdb/client/impl/SenderSlot.java
+++ b/core/src/main/java/io/questdb/client/impl/SenderSlot.java
@@ -48,6 +48,11 @@ final class SenderSlot {
private final Sender delegate;
private final SenderPool pool;
private final int slotIndex;
+ // Index in SenderPool.retiredSlots, or -1 while this slot is not retired.
+ // Guarded by the pool lock. The pool maintains this index across swap
+ // removals so the delegate's release callback can recover this exact slot
+ // without scanning every other retired slot.
+ private int retiredIndex = -1;
// Monotonic lease id. Mutated only under the SenderPool lock (bumped in
// borrow() when the slot is handed out and in giveBack()/discardBroken()
// when it is returned). A PooledSender wrapper captures it live for its
@@ -112,6 +117,14 @@ SenderPool pool() {
return pool;
}
+ int retiredIndex() {
+ return retiredIndex;
+ }
+
+ void retiredIndex(int retiredIndex) {
+ this.retiredIndex = retiredIndex;
+ }
+
int slotIndex() {
return slotIndex;
}
diff --git a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java
index c1b1eec7..b5e43a35 100644
--- a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java
+++ b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java
@@ -149,6 +149,14 @@ public void close() {
}
}
+ @Override
+ public void closeTraffic() {
+ // A concurrent TLS send may still use the SSLEngine and direct buffers.
+ // Shut down only the delegate traffic path here; close() releases the
+ // retained fd and frees TLS state after the owning worker has stopped.
+ delegate.closeTraffic();
+ }
+
@Override
public int getFd() {
return delegate.getFd();
diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java
index f649d330..b9e669d0 100644
--- a/core/src/main/java/io/questdb/client/network/Net.java
+++ b/core/src/main/java/io/questdb/client/network/Net.java
@@ -151,6 +151,8 @@ public static void init() {
public static native int setTcpNoDelay(int fd, boolean noDelay);
+ public static native int shutdown(int fd);
+
public static long sockaddr(int ipv4address, int port) {
SOCK_ADDR_COUNTER.incrementAndGet();
return sockaddr0(ipv4address, port);
diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacade.java b/core/src/main/java/io/questdb/client/network/NetworkFacade.java
index d23824a5..4c66f0df 100644
--- a/core/src/main/java/io/questdb/client/network/NetworkFacade.java
+++ b/core/src/main/java/io/questdb/client/network/NetworkFacade.java
@@ -78,6 +78,15 @@ public interface NetworkFacade {
int setTcpNoDelay(int fd, boolean noDelay);
+ /**
+ * Shuts down traffic on a descriptor owned by this facade without releasing
+ * the descriptor. Custom facades must override this method rather than let
+ * native code operate on a synthetic or remapped descriptor.
+ */
+ default int shutdown(int fd) {
+ throw new UnsupportedOperationException("traffic shutdown is not supported by this network facade");
+ }
+
long sockaddr(int address, int port);
int socketTcp(boolean blocking);
diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java
index 64ea0dc7..1d1d7e20 100644
--- a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java
+++ b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java
@@ -132,6 +132,11 @@ public int setTcpNoDelay(int fd, boolean noDelay) {
return Net.setTcpNoDelay(fd, noDelay);
}
+ @Override
+ public int shutdown(int fd) {
+ return Net.shutdown(fd);
+ }
+
@Override
public long sockaddr(int address, int port) {
return Net.sockaddr(address, port);
diff --git a/core/src/main/java/io/questdb/client/network/PlainSocket.java b/core/src/main/java/io/questdb/client/network/PlainSocket.java
index 555affd2..cc175f51 100644
--- a/core/src/main/java/io/questdb/client/network/PlainSocket.java
+++ b/core/src/main/java/io/questdb/client/network/PlainSocket.java
@@ -29,7 +29,7 @@
public class PlainSocket implements Socket {
private final Logger log;
private final NetworkFacade nf;
- private int fd = -1;
+ private volatile int fd = -1;
public PlainSocket(NetworkFacade nf, Logger log) {
this.nf = nf;
@@ -37,13 +37,22 @@ public PlainSocket(NetworkFacade nf, Logger log) {
}
@Override
- public void close() {
+ public synchronized void close() {
if (fd != -1) {
nf.close(fd, log);
fd = -1;
}
}
+ @Override
+ public synchronized void closeTraffic() {
+ if (fd != -1 && nf.shutdown(fd) != 0) {
+ throw new IllegalStateException(
+ "could not shut down socket traffic [fd=" + fd + ", errno=" + nf.errno() + ']'
+ );
+ }
+ }
+
@Override
public int getFd() {
return fd;
diff --git a/core/src/main/java/io/questdb/client/network/Socket.java b/core/src/main/java/io/questdb/client/network/Socket.java
index 0cdce517..6fceb30b 100644
--- a/core/src/main/java/io/questdb/client/network/Socket.java
+++ b/core/src/main/java/io/questdb/client/network/Socket.java
@@ -37,6 +37,17 @@
public interface Socket extends QuietCloseable {
int WRITE_FLAG = 1;
+ /**
+ * Closes only the network traffic path so a concurrent blocking send or
+ * receive returns. Implementations must retain any I/O buffers until the
+ * owning worker has stopped and {@link #close()} performs full cleanup.
+ * The compatibility default fails without touching implementation-owned
+ * resources; custom sockets must override this method to opt in.
+ */
+ default void closeTraffic() {
+ throw new UnsupportedOperationException("traffic shutdown is not supported by this socket");
+ }
+
/**
* @return file descriptor associated with the socket.
*/
diff --git a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java
index dca93e84..194d0972 100644
--- a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java
+++ b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java
@@ -86,6 +86,11 @@ public int fsync(int fd) {
return Files.fsync(fd);
}
+ @Override
+ public int fsyncDir(String dir) {
+ return Files.fsyncDir(dir);
+ }
+
@Override
public long length(int fd) {
return Files.length(fd);
@@ -106,6 +111,21 @@ public int mkdir(String path, int mode) {
return Files.mkdir(path, mode);
}
+ @Override
+ public long mmap(int fd, long len, long offset, int flags, int memoryTag) {
+ return Files.mmap(fd, len, offset, flags, memoryTag);
+ }
+
+ @Override
+ public int msync(long addr, long len, boolean async) {
+ return Files.msync(addr, len, async);
+ }
+
+ @Override
+ public void munmap(long address, long len, int memoryTag) {
+ Files.munmap(address, len, memoryTag);
+ }
+
@Override
public int openCleanRW(String path) {
return Files.openCleanRW(path);
@@ -126,6 +146,16 @@ public int openRW(long pathPtr) {
return Files.openRW(pathPtr);
}
+ @Override
+ public int openRWExclusive(String path) {
+ return Files.openRWExclusive(path);
+ }
+
+ @Override
+ public int openRWExclusive(long pathPtr) {
+ return Files.openRWExclusive(pathPtr);
+ }
+
@Override
public long length(long pathPtr) {
return Files.length(pathPtr);
diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java
index 02756c37..b0fdfb2e 100644
--- a/core/src/main/java/io/questdb/client/std/Files.java
+++ b/core/src/main/java/io/questdb/client/std/Files.java
@@ -154,6 +154,24 @@ public static int openRW(long pathPtr) {
return openRW0(pathPtr);
}
+ /**
+ * Atomically creates {@code path} for read-write access. Fails with -1
+ * when the path already exists; existing bytes are never truncated.
+ */
+ public static int openRWExclusive(String path) {
+ long ptr = pathPtr(path);
+ try {
+ return openRWExclusive0(ptr);
+ } finally {
+ freePathPtr(ptr);
+ }
+ }
+
+ /** Native-path variant of {@link #openRWExclusive(String)}. */
+ public static int openRWExclusive(long pathPtr) {
+ return openRWExclusive0(pathPtr);
+ }
+
/**
* Opens {@code path} for append-only writes, creating it (mode 0644) if
* absent. Every {@link #append(int, long, long)} writes at end-of-file
@@ -408,6 +426,20 @@ public static String utf8ToString(long nameZ) {
*/
public static native int fsync(int fd);
+ /**
+ * Forces directory-entry updates under {@code dir} to durable storage.
+ * Returns 0 on success and non-zero on failure. Callers use this after
+ * unlinking files whose absence must survive a host crash.
+ */
+ public static int fsyncDir(String dir) {
+ long ptr = pathPtr(dir);
+ try {
+ return fsyncDir0(ptr);
+ } finally {
+ freePathPtr(ptr);
+ }
+ }
+
/**
* Truncates the file to exactly {@code size} bytes via {@code ftruncate}.
* Returns {@code true} on success. Does NOT reserve disk space — the
@@ -555,10 +587,14 @@ public static void munmap(long address, long len, int memoryTag) {
static native int close0(int fd);
+ static native int fsyncDir0(long lpszName);
+
static native int openRO0(long lpszName);
static native int openRW0(long lpszName);
+ static native int openRWExclusive0(long lpszName);
+
static native int openAppend0(long lpszName);
static native int openCleanRW0(long lpszName);
diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java
index 1b408cf4..082e680e 100644
--- a/core/src/main/java/io/questdb/client/std/FilesFacade.java
+++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java
@@ -85,6 +85,10 @@ public interface FilesFacade {
int fsync(int fd);
+ default int fsyncDir(String dir) {
+ return Files.fsyncDir(dir);
+ }
+
long length(int fd);
/**
@@ -107,6 +111,18 @@ public interface FilesFacade {
int mkdir(String path, int mode);
+ default long mmap(int fd, long len, long offset, int flags, int memoryTag) {
+ return Files.mmap(fd, len, offset, flags, memoryTag);
+ }
+
+ default int msync(long addr, long len, boolean async) {
+ return Files.msync(addr, len, async);
+ }
+
+ default void munmap(long address, long len, int memoryTag) {
+ Files.munmap(address, len, memoryTag);
+ }
+
int openCleanRW(String path);
/**
@@ -122,6 +138,14 @@ public interface FilesFacade {
/** Variant of {@link #openRW(String)} taking a pre-encoded native UTF-8 path pointer. */
int openRW(long pathPtr);
+ default int openRWExclusive(String path) {
+ return Files.openRWExclusive(path);
+ }
+
+ default int openRWExclusive(long pathPtr) {
+ return Files.openRWExclusive(pathPtr);
+ }
+
/**
* Variant of {@code length(String)} taking a pre-encoded native UTF-8 path
* pointer; same allocation-elision rationale as {@link #openRW(long)}.
diff --git a/core/src/main/java/io/questdb/client/std/ObjList.java b/core/src/main/java/io/questdb/client/std/ObjList.java
index 9e0e7b29..34d989ae 100644
--- a/core/src/main/java/io/questdb/client/std/ObjList.java
+++ b/core/src/main/java/io/questdb/client/std/ObjList.java
@@ -149,7 +149,7 @@ public void remove(int from, int to) {
System.arraycopy(buffer, to + 1, buffer, from, move);
}
pos = Math.max(0, pos - (to - from + 1));
- Arrays.fill(buffer, pos, buffer.length - 1, null);
+ Arrays.fill(buffer, pos, buffer.length, null);
}
public void remove(int index) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java
index 9f1500f2..e4fb86c5 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java
@@ -35,8 +35,10 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.file.Paths;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
/**
* Regression tests for the close() drain semantics.
@@ -49,6 +51,83 @@
*/
public class CloseDrainTest {
+ @Test(timeout = 30_000L)
+ public void testCloseBlocksAcrossAllReplicaWindowUntilPromotion() throws Exception {
+ DelayingAckHandler handler = new DelayingAckHandler(0);
+ try (TestWebSocketServer server = new TestWebSocketServer(handler, false, "PRIMARY")) {
+ server.setRejectWithRole("REPLICA");
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String cfg = "ws::addr=localhost:" + server.getPort()
+ + ";initial_connect_retry=async"
+ + ";reconnect_initial_backoff_millis=20"
+ + ";reconnect_max_backoff_millis=100"
+ + ";close_flush_timeout_millis=10000;";
+ QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg);
+ CountDownLatch closeDrainWaiting = new CountDownLatch(1);
+ CountDownLatch releaseCloseDrain = new CountDownLatch(1);
+ AtomicReference closeFailure = new AtomicReference<>();
+ AtomicReference hookFailure = new AtomicReference<>();
+ sender.setCloseDrainWaitingHook(() -> {
+ closeDrainWaiting.countDown();
+ try {
+ if (!releaseCloseDrain.await(10, TimeUnit.SECONDS)) {
+ throw new AssertionError("promotion did not release the close-drain witness");
+ }
+ } catch (Throwable t) {
+ hookFailure.set(t);
+ }
+ });
+ sender.table("foo").longColumn("v", 1L).atNow();
+ sender.flush();
+
+ Thread closer = new Thread(() -> {
+ try {
+ sender.close();
+ } catch (Throwable t) {
+ closeFailure.set(t);
+ }
+ }, "all-replica-close-drain");
+ try {
+ closer.start();
+ Assert.assertTrue("server never produced the all-replica role rejection",
+ server.awaitRoleReject(5, TimeUnit.SECONDS));
+ Assert.assertTrue("close never observed its real unacknowledged drain target",
+ closeDrainWaiting.await(5, TimeUnit.SECONDS));
+
+ Assert.assertTrue("pre-promotion witness must include a role rejection",
+ server.roleRejectCount() >= 1);
+ Assert.assertEquals("pre-promotion data must remain unacknowledged",
+ -1L, sender.getAckedFsn());
+ Assert.assertTrue("close must remain pending for the whole all-replica window",
+ closer.isAlive());
+ Assert.assertEquals("promotion must not have delivered or replayed the frame yet",
+ 0L, handler.nextSeq.get());
+
+ // The close-drain barrier has held continuously since it observed
+ // targetFsn > ackedFsn. Promote first, then release the barrier:
+ // completion therefore cannot precede the deterministic recovery event.
+ server.setAdvertisedRole("PRIMARY");
+ server.setRejectWithRole(null);
+ releaseCloseDrain.countDown();
+ closer.join(10_000L);
+
+ Assert.assertFalse("close did not complete after promotion", closer.isAlive());
+ Assert.assertNull("close-drain witness failed", hookFailure.get());
+ Assert.assertNull("close failed after promotion", closeFailure.get());
+ Assert.assertEquals("the unacknowledged frame must be delivered exactly once",
+ 1L, handler.nextSeq.get());
+ } finally {
+ server.setAdvertisedRole("PRIMARY");
+ server.setRejectWithRole(null);
+ releaseCloseDrain.countDown();
+ closer.join(10_000L);
+ sender.close();
+ }
+ }
+ }
+
@Test
public void testCloseBlocksUntilAckArrives() throws Exception {
// Server delays every ACK by 800ms. With the default
@@ -77,6 +156,56 @@ public void testCloseBlocksUntilAckArrives() throws Exception {
}
}
+ @Test
+ public void testCloseStartedHookRunsAfterClosedStateTransition() throws Exception {
+ QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1);
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicReference closerFailure = new AtomicReference<>();
+ AtomicReference hookFailure = new AtomicReference<>();
+ sender.setCloseStartedHook(() -> {
+ try {
+ try {
+ sender.table("must_reject_after_close_started");
+ Assert.fail("close-started hook ran before closed=true was published");
+ } catch (LineSenderException expected) {
+ // Required lifecycle boundary: public operations already reject.
+ }
+ entered.countDown();
+ if (!release.await(10, TimeUnit.SECONDS)) {
+ throw new AssertionError("close-started hook release timed out");
+ }
+ } catch (Throwable t) {
+ hookFailure.set(t);
+ entered.countDown();
+ }
+ });
+
+ Assert.assertEquals("installing the hook must not emit a pre-invocation witness",
+ 1L, entered.getCount());
+ Thread closer = new Thread(() -> {
+ try {
+ sender.close();
+ } catch (Throwable t) {
+ closerFailure.set(t);
+ }
+ }, "close-started-hook-test");
+ try {
+ closer.start();
+ Assert.assertTrue("close() never reached its internal lifecycle witness",
+ entered.await(10, TimeUnit.SECONDS));
+ Assert.assertTrue("close() completed while its internal witness was held",
+ closer.isAlive());
+ } finally {
+ release.countDown();
+ closer.join(10_000L);
+ sender.close();
+ }
+ Assert.assertFalse("close thread did not finish", closer.isAlive());
+ Assert.assertNull("close-started hook failed", hookFailure.get());
+ Assert.assertNull("close() failed", closerFailure.get());
+ }
+
@Test
public void testCloseFastWhenTimeoutIsZero() throws Exception {
// Same delayed-ACK server, but with close_flush_timeout_millis=0
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java
index 173374ec..5f04ca68 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java
@@ -27,6 +27,7 @@
import io.questdb.client.Sender;
import io.questdb.client.SenderError;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode;
import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
@@ -43,8 +44,10 @@
import java.nio.charset.StandardCharsets;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongSupplier;
/**
* Behavior of {@code initial_connect_retry=async}: the producer-thread
@@ -205,6 +208,78 @@ public void testAsyncDeliversBufferedRowsWhenServerArrivesLate() throws Exceptio
}
}
+ @Test
+ public void testAsyncMetricsDistinguishInitialSendFromReconnectReplay() throws Exception {
+ ReplayMetricsHandler handler = new ReplayMetricsHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ String cfg = "ws::addr=localhost:" + server.getPort()
+ + sfDirOpt() + ";initial_connect_retry=async"
+ + ";reconnect_initial_backoff_millis=20"
+ + ";reconnect_max_backoff_millis=200"
+ + ";close_flush_timeout_millis=2000;";
+ Sender sender = Sender.fromConfig(cfg);
+ try {
+ QwpWebSocketSender wss = (QwpWebSocketSender) sender;
+ handler.bind(wss, server);
+
+ // Publish before the server starts accepting. The frame has
+ // never been on any wire, so its eventual first delivery must
+ // increase sent, but not replayed.
+ sender.table("foo").longColumn("v", 1L).atNow();
+ sender.flush();
+ awaitAtLeastOneConnectAttempt(wss);
+ Assert.assertEquals("the server must not handshake before start", 0, server.handshakeCount());
+
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ Assert.assertTrue("the initial frame must be ACKed",
+ handler.awaitInitialAck(5, TimeUnit.SECONDS));
+ awaitMetricAtLeast("initial ACK", wss::getTotalAcks, 1L);
+ Assert.assertEquals("exactly one frame must be sent initially",
+ 1L, wss.getTotalFramesSent());
+ long replayedAfterInitialDelivery = wss.getTotalFramesReplayed();
+
+ // The second frame reaches the established connection, but the
+ // fixture closes that connection without ACKing it. It must then
+ // arrive again on a genuinely new connection and count once as
+ // replayed.
+ sender.table("foo").longColumn("v", 2L).atNow();
+ sender.flush();
+ Assert.assertTrue("the fixture must close after the unacked frame",
+ handler.awaitDisconnect(5, TimeUnit.SECONDS));
+ Assert.assertTrue("the reconnect must reach the gated server",
+ server.awaitRoleReject(5, TimeUnit.SECONDS));
+
+ // Publish another frame while reconnect is blocked at a role
+ // reject, then reopen the gate. This frame has never been sent
+ // and must not inflate the replay counter after reconnect.
+ sender.table("foo").longColumn("v", 3L).atNow();
+ sender.flush();
+ server.setRejectWithRole(null);
+
+ Assert.assertTrue("the unacked frame must be replayed",
+ handler.awaitReplay(5, TimeUnit.SECONDS));
+ Assert.assertTrue("the outage-queued frame must be delivered",
+ handler.awaitOutageQueuedDelivery(5, TimeUnit.SECONDS));
+ awaitMetricAtLeast("replayed frame", wss::getTotalFramesReplayed, 1L);
+ awaitMetricAtLeast("post-reconnect ACKs", wss::getTotalAcks, 3L);
+
+ Assert.assertTrue("replay must arrive on a new WebSocket connection",
+ handler.wasReplayOnNewConnection());
+ Assert.assertTrue("the server must observe a reconnect",
+ server.handshakeCount() >= 2);
+ Assert.assertEquals("initial delivery is not a replay",
+ 0L, replayedAfterInitialDelivery);
+ Assert.assertEquals("three first sends plus one genuine resend",
+ 4L, wss.getTotalFramesSent());
+ Assert.assertEquals("only the genuine resend is replayed",
+ 1L, wss.getTotalFramesReplayed());
+ } finally {
+ closeQuietly(sender);
+ }
+ }
+ }
+
@Test
public void testAsyncReturnsImmediatelyWithNoServer() {
// No server. With async mode, fromConfig must return fast — the
@@ -345,6 +420,17 @@ private static void awaitAtLeastOneConnectAttempt(QwpWebSocketSender wss) {
* before the budget expired would satisfy it even if the loop then
* exited.
*/
+ private static void awaitMetricAtLeast(String label, LongSupplier metric, long expected) {
+ long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (metric.getAsLong() < expected) {
+ if (System.nanoTime() > deadlineNanos) {
+ throw new AssertionError(label + " did not reach " + expected
+ + " within 5s; current=" + metric.getAsLong());
+ }
+ io.questdb.client.std.Compat.onSpinWait();
+ }
+ }
+
private static void awaitReconnectAttemptsAdvance(QwpWebSocketSender wss) {
long snapshot = wss.getTotalReconnectAttempts();
long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
@@ -476,6 +562,83 @@ public void onError(@NotNull SenderError err) {
}
}
+ private static class ReplayMetricsHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final CountDownLatch disconnect = new CountDownLatch(1);
+ private final CountDownLatch initialAck = new CountDownLatch(1);
+ private final CountDownLatch outageQueuedDelivery = new CountDownLatch(1);
+ private final AtomicLong received = new AtomicLong();
+ private final CountDownLatch replay = new CountDownLatch(1);
+ private final AtomicBoolean replayOnNewConnection = new AtomicBoolean();
+ private TestWebSocketServer.ClientHandler initialClient;
+ private volatile TestWebSocketServer server;
+ private volatile QwpWebSocketSender sender;
+
+ boolean awaitDisconnect(long timeout, TimeUnit unit) throws InterruptedException {
+ return disconnect.await(timeout, unit);
+ }
+
+ boolean awaitInitialAck(long timeout, TimeUnit unit) throws InterruptedException {
+ return initialAck.await(timeout, unit);
+ }
+
+ boolean awaitOutageQueuedDelivery(long timeout, TimeUnit unit) throws InterruptedException {
+ return outageQueuedDelivery.await(timeout, unit);
+ }
+
+ boolean awaitReplay(long timeout, TimeUnit unit) throws InterruptedException {
+ return replay.await(timeout, unit);
+ }
+
+ void bind(QwpWebSocketSender sender, TestWebSocketServer server) {
+ this.sender = sender;
+ this.server = server;
+ }
+
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ long ordinal = received.incrementAndGet();
+ try {
+ if (ordinal == 1L) {
+ initialClient = client;
+ client.sendBinary(AckHandler.buildAck(0L));
+ initialAck.countDown();
+ } else if (ordinal == 2L) {
+ // Receipt alone can race the sender's post-send metric
+ // increments. Wait until sendBinary returned before closing,
+ // proving this frame completed a send and must be replayed.
+ awaitSentFrames(2L);
+ server.setRejectWithRole("REPLICA");
+ client.sendClose(WebSocketCloseCode.GOING_AWAY, "test reconnect");
+ disconnect.countDown();
+ } else if (ordinal == 3L) {
+ replayOnNewConnection.set(client != initialClient);
+ client.sendBinary(AckHandler.buildAck(0L));
+ replay.countDown();
+ } else if (ordinal == 4L) {
+ client.sendBinary(AckHandler.buildAck(1L));
+ outageQueuedDelivery.countDown();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ boolean wasReplayOnNewConnection() {
+ return replayOnNewConnection.get();
+ }
+
+ private void awaitSentFrames(long expected) {
+ long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (sender.getTotalFramesSent() < expected) {
+ if (System.nanoTime() > deadlineNanos) {
+ throw new AssertionError("sent-frame metric did not reach " + expected
+ + " before disconnect; current=" + sender.getTotalFramesSent());
+ }
+ io.questdb.client.std.Compat.onSpinWait();
+ }
+ }
+ }
+
/**
* Raw-socket fixture: every accepted connection responds with HTTP
* 401 Unauthorized and closes. Used to drive the async-init
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java
new file mode 100644
index 00000000..5bfb0863
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java
@@ -0,0 +1,141 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+public class QwpWebSocketSenderCursorEngineAttachmentTest {
+
+ private static final long SEGMENT_SIZE = 64 * 1024L;
+
+ @Rule
+ public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build();
+
+ @Test
+ public void testNullDoesNotDetachAnAttachedEngine() throws Exception {
+ assertMemoryLeak(() -> {
+ CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE);
+ QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1);
+ try {
+ sender.setCursorEngine(engine, true);
+ assertSecondAttachmentRejected(sender, null, false);
+ sender.close();
+ Assert.assertTrue("sender must retain ownership after rejected detach",
+ engine.isCloseCompleted());
+ } finally {
+ sender.close();
+ engine.close();
+ }
+ });
+ }
+
+ @Test
+ public void testOwnedEngineCannotBeReplacedAndItsSlotIsReleased() throws Exception {
+ assertMemoryLeak(() -> {
+ File slotDir = new File(sfDir.getRoot(), "owned-slot");
+ CursorSendEngine first = null;
+ CursorSendEngine replacement = null;
+ CursorSendEngine reacquired = null;
+ QwpWebSocketSender sender = null;
+ boolean replacementRejected = false;
+ try {
+ first = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE);
+ replacement = new CursorSendEngine(null, SEGMENT_SIZE);
+ sender = QwpWebSocketSender.createForTesting("localhost", 1);
+ sender.setCursorEngine(first, true);
+ try {
+ sender.setCursorEngine(replacement, true);
+ } catch (LineSenderException e) {
+ replacementRejected = true;
+ Assert.assertTrue(e.getMessage().contains("already attached"));
+ }
+
+ sender.close();
+ sender = null;
+ try {
+ reacquired = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE);
+ } catch (IllegalStateException e) {
+ throw new AssertionError("sender close did not release the first owned "
+ + "engine's slot after a second attachment attempt", e);
+ }
+ Assert.assertTrue("second attachment must be rejected", replacementRejected);
+ } finally {
+ if (sender != null) {
+ sender.close();
+ }
+ if (reacquired != null) {
+ reacquired.close();
+ }
+ if (replacement != null) {
+ replacement.close();
+ }
+ if (first != null) {
+ first.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSameSharedEngineCannotTransferOwnership() throws Exception {
+ assertMemoryLeak(() -> {
+ CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE);
+ QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1);
+ try {
+ sender.setCursorEngine(engine, false);
+ assertSecondAttachmentRejected(sender, engine, true);
+ sender.close();
+ Assert.assertFalse("rejected attachment must not transfer ownership",
+ engine.isCloseCompleted());
+ } finally {
+ sender.close();
+ engine.close();
+ }
+ });
+ }
+
+ private static void assertSecondAttachmentRejected(
+ QwpWebSocketSender sender,
+ CursorSendEngine engine,
+ boolean takeOwnership
+ ) {
+ try {
+ sender.setCursorEngine(engine, takeOwnership);
+ Assert.fail("expected second cursor engine attachment to be rejected");
+ } catch (LineSenderException e) {
+ Assert.assertTrue(e.getMessage().contains("already attached"));
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java
index 85522e9d..63553ebc 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java
@@ -24,9 +24,22 @@
package io.questdb.client.test.cutlass.qwp.client;
+import io.questdb.client.DefaultHttpClientConfiguration;
import io.questdb.client.Sender;
+import io.questdb.client.SenderConnectionEvent;
+import io.questdb.client.SenderError;
+import io.questdb.client.cutlass.http.client.WebSocketClient;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock;
+import io.questdb.client.network.PlainSocketFactory;
+import io.questdb.client.std.Files;
import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
import io.questdb.client.test.tools.TestUtils;
import org.junit.Assert;
@@ -36,8 +49,12 @@
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.nio.file.Paths;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.LockSupport;
/**
@@ -46,9 +63,13 @@
*
*
Happy path — a clean {@code close()} that winds the I/O thread
* down reports {@code isSlotLockReleased() == true}.
- *
Leak path — a {@code close()} that early-returns because the I/O
- * thread refused to stop ({@code ioThreadStopped == false}) reports
+ *
Leak path — a {@code close()} that retains the lock because an
+ * I/O or manager worker did not stop reports
* {@code isSlotLockReleased() == false}.
+ *
Recovery path — the getter is monotonic but not frozen: it
+ * re-probes the retained engine, so once the deferred cleanup (worker
+ * exit path or a retried close) releases the flock, the getter flips to
+ * {@code true} and a pool that retired the slot can recover it.
*
* The pool's {@code flockReleased(s)} treats {@code false} as "flock still held,
* retire the slot permanently". Before this test that contract was driven only
@@ -178,6 +199,380 @@ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception
});
}
+ /**
+ * Manager-worker handoff path: engine close retains the slot while a shared
+ * manager is still inside a service pass. The sender must expose that
+ * retained flock to SenderPool. Repeated sender close calls remain no-ops;
+ * the manager pass owns deferred cleanup and releases the flock when it
+ * finishes, so the sender can expose recovery without a direct close retry.
+ */
+ @Test(timeout = 30_000L)
+ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-slot-lock-manager-" + System.nanoTime()).toString();
+ Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT));
+ String slot = tmpDir + "/slot";
+ long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L;
+ SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60));
+ CountDownLatch cleanupFinished = new CountDownLatch(1);
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean fired = new AtomicBoolean();
+ AtomicReference hookErr = new AtomicReference<>();
+ CursorSendEngine engine = null;
+ QwpWebSocketSender wss = null;
+ boolean managerClosed = false;
+ try {
+ manager.setAfterRingCleanupHook(cleanupFinished::countDown);
+ manager.setBeforeInstallSyncHook(() -> {
+ if (!fired.compareAndSet(false, true)) return;
+ workerBlocked.countDown();
+ try {
+ if (!releaseWorker.await(20, TimeUnit.SECONDS)) {
+ hookErr.compareAndSet(null,
+ new AssertionError("timed out waiting for test to release worker"));
+ }
+ } catch (Throwable t) {
+ hookErr.compareAndSet(null, t);
+ }
+ });
+ manager.start();
+ engine = new CursorSendEngine(slot, segSize, manager);
+ Assert.assertTrue("worker never reached install hook",
+ workerBlocked.await(5, TimeUnit.SECONDS));
+
+ wss = QwpWebSocketSender.createForTesting("localhost", 1);
+ wss.setCursorEngine(engine, true);
+ manager.setWorkerJoinTimeoutMillis(50L);
+ wss.close();
+
+ Assert.assertFalse("sender reported a retained flock as released",
+ wss.isSlotLockReleased());
+ Assert.assertFalse("engine close must remain incomplete while worker is in service",
+ engine.isCloseCompleted());
+ try {
+ SlotLock probe = SlotLock.acquire(slot);
+ probe.close();
+ Assert.fail("slot became acquirable while manager worker was still in service");
+ } catch (Exception expected) {
+ // good — the incomplete close retained the flock.
+ }
+
+ // Sender.close() is idempotent: a second call must not retry
+ // or change ownership while the manager worker is still live.
+ wss.close();
+ Assert.assertFalse("repeated close changed retained-flock reporting",
+ wss.isSlotLockReleased());
+ Assert.assertFalse("repeated close unexpectedly retried engine cleanup",
+ engine.isCloseCompleted());
+
+ releaseWorker.countDown();
+ Assert.assertTrue("shared-manager pass did not finish deferred cleanup",
+ cleanupFinished.await(5, TimeUnit.SECONDS));
+ Assert.assertTrue("deferred cleanup did not complete after the ring pass",
+ engine.isCloseCompleted());
+ // Recovery contract: the sender re-probes its retained engine,
+ // so the completed cleanup (and released flock) MUST become
+ // visible — this is what lets SenderPool recover a slot it
+ // had already retired as leaked.
+ Assert.assertTrue("sender must expose the late flock release to the pool",
+ wss.isSlotLockReleased());
+ try (SlotLock probe = SlotLock.acquire(slot)) {
+ Assert.assertNotNull("slot must be acquirable after deferred cleanup", probe);
+ }
+
+ manager.close();
+ managerClosed = true;
+ if (hookErr.get() != null) {
+ throw new AssertionError("install hook failed", hookErr.get());
+ }
+ } finally {
+ manager.setAfterRingCleanupHook(null);
+ manager.setBeforeInstallSyncHook(null);
+ releaseWorker.countDown();
+ if (wss != null) {
+ try {
+ wss.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ if (!managerClosed) {
+ manager.close();
+ }
+ rmDirRecursive(tmpDir);
+ Files.remove(tmpDir);
+ }
+ });
+ }
+
+ /**
+ * Delegated-I/O-close recovery path: when {@code close()} bails early
+ * because the I/O thread refuses to stop, the engine close is delegated
+ * to that thread's exit path ({@code delegateEngineClose()} returned
+ * true) and the sender must retain the engine for re-probing —
+ * {@code isSlotLockReleased()} stays false while the thread lives, then
+ * flips true the moment the thread's exit path completes the engine
+ * close and releases the flock. Pre-fix, this branch discarded the
+ * engine reference, so the late release was permanently invisible to a
+ * pool that had retired the slot. The existing forged I/O-refusal test
+ * can never reach this branch: its sabotaged loop throws from
+ * {@code delegateEngineClose()} itself (nulled latch), before the
+ * retained-engine assignment.
+ *
+ * The wedge is the same deterministic one CursorWebSocketSendLoop's C5
+ * test uses: the I/O thread sits in a blocking "connect" (a
+ * ReconnectFactory immune to unpark, never interrupted by loop.close()),
+ * and the closer thread's pre-set interrupt flag makes
+ * {@code shutdownLatch.await()} throw immediately — the real production
+ * path to {@code ioThreadStopped = false}.
+ */
+ @Test(timeout = 30_000L)
+ public void testDelegatedIoThreadEngineCloseFlipsSlotLockReleased() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-slot-lock-delegated-" + System.nanoTime()).toString();
+ Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT));
+ String slot = tmpDir + "/slot";
+ long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L;
+ final CountDownLatch enteredConnect = new CountDownLatch(1);
+ final CountDownLatch releaseConnect = new CountDownLatch(1);
+ final AtomicReference ioThreadRef = new AtomicReference<>();
+ final StubWebSocketClient stubClient = new StubWebSocketClient();
+ // Healthy owned manager: once the wedged I/O thread is released,
+ // its exit-path engine.close() completes normally and releases
+ // the flock — the flip this test pins.
+ CursorSendEngine engine = new CursorSendEngine(slot, segSize);
+ CursorWebSocketSendLoop loop = null;
+ QwpWebSocketSender wss = null;
+ try {
+ // Stand-in for a blocking native connect(2): entered by the
+ // loop's I/O thread, immune to unpark, never interrupted by
+ // loop.close().
+ CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> {
+ ioThreadRef.set(Thread.currentThread());
+ enteredConnect.countDown();
+ releaseConnect.await();
+ return stubClient;
+ };
+ loop = new CursorWebSocketSendLoop(
+ null /* async-initial-connect: the I/O thread drives the connect */,
+ engine, 0L, 1_000L,
+ stuckConnect,
+ 5_000L, 100L, 5_000L, false);
+ loop.start();
+ Assert.assertTrue("I/O thread never reached the connect factory",
+ enteredConnect.await(5, TimeUnit.SECONDS));
+
+ wss = QwpWebSocketSender.createForTesting("localhost", 1);
+ wss.setCursorEngine(engine, true);
+ setField(wss, "cursorSendLoop", loop);
+
+ // Drive the real early-bail close() on a thread whose pending
+ // interrupt lands in loop.close()'s shutdownLatch.await().
+ AtomicReference closeFailure = new AtomicReference<>();
+ QwpWebSocketSender wssRef = wss;
+ Thread closer = new Thread(() -> {
+ Thread.currentThread().interrupt();
+ try {
+ wssRef.close();
+ } catch (Throwable t) {
+ closeFailure.set(t);
+ }
+ }, "delegated-close-closer");
+ closer.setDaemon(true);
+ closer.start();
+ closer.join(TimeUnit.SECONDS.toMillis(10));
+ Assert.assertFalse("closer thread did not finish", closer.isAlive());
+ Assert.assertNotNull("close() must surface the failed I/O-thread stop",
+ closeFailure.get());
+
+ // The I/O thread is still wedged: the flock is retained and
+ // must be reported retained.
+ Assert.assertFalse(
+ "isSlotLockReleased() must be false while the delegated engine "
+ + "close is pending on the wedged I/O thread",
+ wss.isSlotLockReleased());
+ Assert.assertFalse("engine close must not have run yet",
+ engine.isCloseCompleted());
+ try {
+ SlotLock probe = SlotLock.acquire(slot);
+ probe.close();
+ Assert.fail("slot became acquirable while the delegated engine close "
+ + "was still pending");
+ } catch (Exception expected) {
+ // good — the flock is genuinely held.
+ }
+
+ // Un-wedge the connect. The I/O thread exits; its exit path
+ // runs the delegated engine.close(), which releases the flock.
+ releaseConnect.countDown();
+ Thread ioThread = ioThreadRef.get();
+ Assert.assertNotNull(ioThread);
+ ioThread.join(TimeUnit.SECONDS.toMillis(10));
+ Assert.assertFalse("I/O thread did not exit after the connect returned",
+ ioThread.isAlive());
+
+ // The recovery contract under test: the getter re-probes the
+ // retained engine, so the late release MUST become visible —
+ // this is what lets SenderPool recover the retired slot.
+ Assert.assertTrue("delegated engine close must have completed on the "
+ + "I/O thread's exit path",
+ engine.isCloseCompleted());
+ Assert.assertTrue(
+ "isSlotLockReleased() must flip true once the delegated engine "
+ + "close released the flock — otherwise the pool retires the "
+ + "slot's capacity until process exit",
+ wss.isSlotLockReleased());
+ try (SlotLock probe = SlotLock.acquire(slot)) {
+ Assert.assertNotNull("slot must be acquirable after the delegated close", probe);
+ }
+ } finally {
+ releaseConnect.countDown();
+ // Reap the loop's bookkeeping now that the I/O thread is gone
+ // (close() threw mid-teardown, so ioThread was left set).
+ Thread.interrupted();
+ if (loop != null) {
+ try {
+ loop.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ if (engine != null && !engine.isCloseCompleted()) {
+ try {
+ engine.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ // The early-return close() deliberately leaked the resources
+ // the (then-running) I/O thread might touch; free the same set
+ // the post-guard tail would have freed.
+ if (wss != null) {
+ freeFieldQuietly(wss, "buffer0");
+ freeFieldQuietly(wss, "buffer1");
+ freeFieldQuietly(wss, "client");
+ freeFieldQuietly(wss, "errorDispatcher");
+ freeFieldQuietly(wss, "progressDispatcher");
+ freeFieldQuietly(wss, "connectionDispatcher");
+ }
+ stubClient.close();
+ rmDirRecursive(tmpDir);
+ Files.remove(tmpDir);
+ }
+ });
+ }
+
+ @Test(timeout = 30_000L)
+ public void testFailedIoStopReclaimsSenderResourcesAfterWorkerExit() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-slot-lock-full-cleanup-" + System.nanoTime()).toString();
+ Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT));
+ String slot = tmpDir + "/slot";
+ long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L;
+ CountDownLatch enteredConnect = new CountDownLatch(1);
+ CountDownLatch releaseConnect = new CountDownLatch(1);
+ AtomicReference ioThreadRef = new AtomicReference<>();
+ StubWebSocketClient loopClient = new StubWebSocketClient();
+ StubWebSocketClient senderClient = new StubWebSocketClient();
+ CursorSendEngine engine = new CursorSendEngine(slot, segSize);
+ CursorWebSocketSendLoop loop = null;
+ try {
+ CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> {
+ ioThreadRef.set(Thread.currentThread());
+ enteredConnect.countDown();
+ releaseConnect.await();
+ return loopClient;
+ };
+ loop = new CursorWebSocketSendLoop(
+ null, engine, 0L, 1_000L, stuckConnect,
+ 5_000L, 100L, 5_000L, false);
+ loop.start();
+ Assert.assertTrue("I/O thread never reached the connect factory",
+ enteredConnect.await(5, TimeUnit.SECONDS));
+
+ QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1);
+ sender.setConnectionListener(event -> {
+ });
+ sender.setErrorHandler(error -> {
+ });
+ sender.setProgressHandler(ackedFsn -> {
+ });
+ sender.setClientForTesting(senderClient);
+ sender.setCursorEngine(engine, true);
+ sender.setCursorSendLoopForTesting(loop);
+
+ SenderConnectionDispatcher connectionDispatcher = sender.getConnectionDispatcherForTesting();
+ SenderErrorDispatcher errorDispatcher = sender.getErrorDispatcherForTesting();
+ SenderProgressDispatcher progressDispatcher = sender.getProgressDispatcherForTesting();
+ Assert.assertTrue(connectionDispatcher.offer(new SenderConnectionEvent(
+ SenderConnectionEvent.Kind.DISCONNECTED,
+ null, SenderConnectionEvent.NO_PORT,
+ null, SenderConnectionEvent.NO_PORT,
+ SenderConnectionEvent.NO_ATTEMPT_NUMBER,
+ SenderConnectionEvent.NO_ROUND_NUMBER,
+ null, 0L)));
+ Assert.assertTrue(errorDispatcher.offer(new SenderError(
+ SenderError.Category.UNKNOWN, SenderError.Policy.RETRIABLE,
+ SenderError.NO_STATUS_BYTE, null, SenderError.NO_MESSAGE_SEQUENCE,
+ -1L, -1L, null, 0L)));
+ Assert.assertTrue(progressDispatcher.offer(0L));
+ Thread connectionDispatcherThread = connectionDispatcher.getWorkerThreadForTesting();
+ Thread errorDispatcherThread = errorDispatcher.getWorkerThreadForTesting();
+ Thread progressDispatcherThread = progressDispatcher.getWorkerThreadForTesting();
+ Assert.assertNotNull(connectionDispatcherThread);
+ Assert.assertNotNull(errorDispatcherThread);
+ Assert.assertNotNull(progressDispatcherThread);
+
+ AtomicReference closeFailure = new AtomicReference<>();
+ Thread closer = new Thread(() -> {
+ Thread.currentThread().interrupt();
+ try {
+ sender.close();
+ } catch (Throwable t) {
+ closeFailure.set(t);
+ }
+ }, "full-cleanup-closer");
+ closer.start();
+ closer.join(TimeUnit.SECONDS.toMillis(10));
+ Assert.assertFalse("closer thread did not finish", closer.isAlive());
+ Assert.assertNotNull("close() must surface the failed I/O-thread stop",
+ closeFailure.get());
+
+ releaseConnect.countDown();
+ Thread ioThread = ioThreadRef.get();
+ Assert.assertNotNull(ioThread);
+ ioThread.join(TimeUnit.SECONDS.toMillis(10));
+ Assert.assertFalse("I/O thread did not exit after release", ioThread.isAlive());
+ Assert.assertTrue("complete sender cleanup callback did not finish",
+ sender.isCloseCleanupComplete());
+ Assert.assertTrue("loop-owned WebSocket client was not closed", loopClient.isClosed());
+ Assert.assertTrue("sender-owned WebSocket client was not closed", senderClient.isClosed());
+ Assert.assertFalse("connection dispatcher worker was not reclaimed",
+ connectionDispatcherThread.isAlive());
+ Assert.assertFalse("error dispatcher worker was not reclaimed",
+ errorDispatcherThread.isAlive());
+ Assert.assertFalse("progress dispatcher worker was not reclaimed",
+ progressDispatcherThread.isAlive());
+ Assert.assertTrue("engine cleanup did not complete", engine.isCloseCompleted());
+ try (SlotLock probe = SlotLock.acquire(slot)) {
+ Assert.assertNotNull(probe);
+ }
+ } finally {
+ releaseConnect.countDown();
+ Thread.interrupted();
+ if (loop != null) {
+ try {
+ loop.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ rmDirRecursive(tmpDir);
+ Files.remove(tmpDir);
+ }
+ });
+ }
+
// ------------------------------------------------------------------ utils
private static void freeFieldQuietly(Object target, String name) {
@@ -192,6 +587,28 @@ private static void freeFieldQuietly(Object target, String name) {
}
}
+ private static void rmDirRecursive(String dir) {
+ if (!Files.exists(dir)) return;
+ long find = Files.findFirst(dir);
+ if (find <= 0) return;
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ if (name != null && !".".equals(name) && !"..".equals(name)) {
+ String child = dir + "/" + name;
+ if (!Files.remove(child)) {
+ rmDirRecursive(child);
+ Files.remove(child);
+ }
+ }
+ rc = Files.findNext(find);
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ }
+
private static T readField(Object target, String name, Class type) throws Exception {
Class cls = target.getClass();
while (cls != null) {
@@ -221,6 +638,39 @@ private static void setField(Object target, String name, Object value) throws Ex
throw new NoSuchFieldException(name);
}
+ /**
+ * Minimal concrete {@link WebSocketClient} — never performs I/O. Handed
+ * to the loop by the stuck-connect factory; the loop's exit path closes
+ * it (close is idempotent via the superclass).
+ */
+ private static final class StubWebSocketClient extends WebSocketClient {
+ private final AtomicBoolean isClosed = new AtomicBoolean();
+
+ StubWebSocketClient() {
+ super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE);
+ }
+
+ @Override
+ public void close() {
+ isClosed.set(true);
+ super.close();
+ }
+
+ @Override
+ protected void ioWait(int timeout, int op) {
+ throw new UnsupportedOperationException("stub: no socket");
+ }
+
+ boolean isClosed() {
+ return isClosed.get();
+ }
+
+ @Override
+ protected void setupIoWait() {
+ // no-op
+ }
+ }
+
/** ACKs every binary frame with a running sequence so flush/close drain cleanly. */
private static final class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
private final AtomicLong nextSeq = new AtomicLong();
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java
index 001f62ce..27bef939 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java
@@ -26,6 +26,8 @@
import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
import io.questdb.client.std.Files;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
import io.questdb.client.test.tools.TestUtils;
import org.junit.After;
import org.junit.Before;
@@ -76,6 +78,45 @@ public void testCrossSessionPersistence() throws Exception {
});
}
+ @Test
+ public void testFallsBackFromTornPlausibleHighValue() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ try (AckWatermark watermark = AckWatermark.open(slotDir)) {
+ assertNotNull(watermark);
+ watermark.write(255L);
+ watermark.write(256L);
+ watermark.sync();
+ }
+
+ // Model a torn little-endian 255 -> 256 transition: the high byte
+ // reached storage while the low byte retained 0xff, yielding the
+ // false FSN 511. A recovered ring with publishedFsn=599 cannot
+ // reject that value using its segment ceiling.
+ long publishedFsn = 599L;
+ long corruptFsn = 511L;
+ assertTrue(corruptFsn <= publishedFsn);
+ String path = slotDir + "/" + AckWatermark.FILE_NAME;
+ int fd = Files.openRW(path);
+ assertTrue(fd >= 0);
+ long corruptByte = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT);
+ try {
+ Unsafe.getUnsafe().putByte(corruptByte, (byte) 0xff);
+ long fsnOffset = AckWatermark.FILE_SIZE == 16 ? 8L : 16L;
+ assertEquals(1L, Files.write(fd, corruptByte, 1, fsnOffset));
+ assertEquals(0, Files.fsync(fd));
+ } finally {
+ Files.close(fd);
+ Unsafe.free(corruptByte, 1, MemoryTag.NATIVE_DEFAULT);
+ }
+
+ try (AckWatermark watermark = AckWatermark.open(slotDir)) {
+ assertNotNull(watermark);
+ assertEquals("torn latest record must fall back to the older watermark",
+ 255L, watermark.read());
+ }
+ });
+ }
+
@Test
public void testFreshFileReadsAsInvalid() throws Exception {
// open() creates the file zero-filled, so magic is 0 and read()
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java
new file mode 100644
index 00000000..a9df2b67
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java
@@ -0,0 +1,209 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Regression for the close-time segment cleanup on a fully-drained slot.
+ *
+ * {@code CursorSendEngine.finishClose} unlinks the acknowledged {@code .sfa}
+ * files and then removes the ack watermark. When the unlink fails (transient
+ * I/O error, permission problem), the residual segment files hold rows the
+ * server already acknowledged. If the watermark does not cover them, a
+ * successor engine on the same slot seeds recovery from {@code lowestBase - 1}
+ * and replays every acknowledged row — duplicates on a non-DEDUP table.
+ *
+ * The test injects the unlink failure by dropping write permission on the
+ * slot directory (POSIX: unlink requires a writable parent directory), so it
+ * is skipped on Windows and when permissions are not enforced (root).
+ * The shared {@link SegmentManager} is deliberately never started: no worker
+ * thread exists, so no manager tick can persist the watermark behind the
+ * test's back, and the close-path quiescence barrier is trivially satisfied —
+ * fully deterministic, no timing coordination needed.
+ */
+public class CursorSendEngineCloseUnlinkFailureTest {
+
+ private String tmpDir;
+
+ @Before
+ public void setUp() {
+ tmpDir = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-engine-close-unlink-fault-" + System.nanoTime()).toString();
+ Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT));
+ }
+
+ @After
+ public void tearDown() {
+ if (tmpDir != null) {
+ removeRecursive(tmpDir);
+ }
+ }
+
+ @Test(timeout = 20_000L)
+ public void testFailedCloseTimeUnlinkMustNotExposeAckedFramesToSuccessor() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L;
+ String slot = tmpDir + "/slot";
+ Path slotPath = Paths.get(slot);
+ long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT);
+ SegmentManager manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60));
+ CursorSendEngine pred = null;
+ CursorSendEngine succ = null;
+ boolean slotDirReadOnly = false;
+ try {
+ fill(payload, 32, (byte) 0x33);
+ pred = new CursorSendEngine(slot, segmentSize, manager);
+ Assert.assertEquals(0L, pred.appendBlocking(payload, 32));
+ Assert.assertEquals(0L, pred.publishedFsn());
+ // The server durably acknowledged FSN 0 in this session.
+ Assert.assertTrue(pred.acknowledge(0L));
+ Assert.assertEquals(0L, pred.ackedFsn());
+
+ // Inject a close-time unlink failure: drop write permission on
+ // the slot dir. Prove the injection works with a probe file --
+ // root (and some filesystems) ignore directory permissions.
+ String probePath = slot + "/probe";
+ Assert.assertTrue(java.nio.file.Files.exists(
+ java.nio.file.Files.createFile(Paths.get(probePath))));
+ try {
+ setPermissions(slotPath, "r-xr-xr-x");
+ } catch (UnsupportedOperationException e) {
+ Assume.assumeNoException("POSIX permissions unavailable on this platform", e);
+ }
+ slotDirReadOnly = true;
+ boolean probeRemoved = Files.remove(probePath);
+ if (probeRemoved) {
+ setPermissions(slotPath, "rwxr-xr-x");
+ slotDirReadOnly = false;
+ }
+ Assume.assumeFalse("directory permissions not enforced (running as root?)",
+ probeRemoved);
+
+ // Fully-drained close: tries to unlink the acknowledged
+ // segment files and fails.
+ pred.close();
+ Assert.assertTrue("flock release needs no dir write; close must complete",
+ pred.isCloseCompleted());
+ pred = null;
+ Assert.assertTrue("injected unlink failure must leave the acknowledged segment",
+ Files.exists(slot + "/sf-initial.sfa"));
+
+ // The transient failure clears before the successor arrives.
+ setPermissions(slotPath, "rwxr-xr-x");
+ slotDirReadOnly = false;
+ Files.remove(probePath);
+
+ succ = new CursorSendEngine(slot, segmentSize, manager);
+ Assert.assertTrue(succ.wasRecoveredFromDisk());
+ Assert.assertEquals(0L, succ.publishedFsn());
+ // THE regression: FSN 0 was acknowledged by the server during
+ // the predecessor's session. The successor must not see it as
+ // replayable, or a non-DEDUP table receives duplicate rows.
+ Assert.assertTrue("successor exposes already-acknowledged frames for replay "
+ + "[ackedFsn=" + succ.ackedFsn()
+ + ", publishedFsn=" + succ.publishedFsn() + "]",
+ succ.ackedFsn() >= succ.publishedFsn());
+
+ // The successor's own fully-drained close retries the cleanup
+ // now that the failure has cleared: segments and watermark gone.
+ succ.close();
+ Assert.assertTrue(succ.isCloseCompleted());
+ succ = null;
+ Assert.assertFalse("successor close did not retry the segment unlink",
+ Files.exists(slot + "/sf-initial.sfa"));
+ Assert.assertFalse("watermark must be removed once no segment file remains",
+ Files.exists(slot + "/" + AckWatermark.FILE_NAME));
+ } finally {
+ if (slotDirReadOnly) {
+ try {
+ setPermissions(slotPath, "rwxr-xr-x");
+ } catch (Throwable ignored) {
+ }
+ }
+ if (pred != null) {
+ pred.close();
+ }
+ if (succ != null) {
+ succ.close();
+ }
+ manager.close();
+ Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT);
+ }
+ });
+ }
+
+ private static void fill(long address, int len, byte value) {
+ for (int i = 0; i < len; i++) {
+ Unsafe.getUnsafe().putByte(address + i, value);
+ }
+ }
+
+ private static void removeRecursive(String dir) {
+ long find = Files.findFirst(dir);
+ if (find > 0) {
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ if (name != null && !".".equals(name) && !"..".equals(name)) {
+ String child = dir + "/" + name;
+ if (!Files.remove(child)) {
+ removeRecursive(child);
+ }
+ }
+ rc = Files.findNext(find);
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ }
+ Files.remove(dir);
+ }
+
+ private static void setPermissions(Path path, String posix) throws Exception {
+ Set perms = PosixFilePermissions.fromString(posix);
+ java.nio.file.Files.setPosixFilePermissions(path, perms);
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java
new file mode 100644
index 00000000..a7017961
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java
@@ -0,0 +1,458 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class CursorSendEngineCrashConsistencyTest {
+
+ @Test
+ public void testCloseDurabilityOrderAndPostCleanupSyncFailurePropagation() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ for (int failAt : new int[]{-1, 4}) {
+ String root = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-close-crash-" + failAt + "-" + System.nanoTime()).toString();
+ String slot = root + "/slot";
+ SegmentManager manager = null;
+ CursorSendEngine engine = null;
+ long payload = 0;
+ Throwable failure = null;
+ try {
+ Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT));
+ CrashImageFilesFacade ff = new CrashImageFilesFacade(slot, failAt);
+ long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L;
+ manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60),
+ SegmentManager.UNLIMITED_TOTAL_BYTES, ff);
+ payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT);
+ engine = new CursorSendEngine(slot, segmentSize, manager);
+ Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7);
+ Assert.assertEquals(0L, engine.appendBlocking(payload, 32));
+ Assert.assertTrue(engine.acknowledge(0L));
+ ff.beginClose();
+ try {
+ engine.close();
+ if (failAt >= 0) {
+ Assert.fail("sync failure was swallowed at boundary " + failAt);
+ }
+ } catch (IllegalStateException expected) {
+ Assert.assertTrue("unexpected close failure: " + expected, failAt >= 0);
+ }
+ engine = null;
+
+ Assert.assertFalse("simulated crash replays acknowledged rows at boundary " + failAt,
+ ff.durableSegments && !ff.durableWatermark);
+ if (failAt == -1) {
+ Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync",
+ "dir-fsync", "segment-remove", "dir-fsync", "watermark-remove"),
+ ff.events);
+ }
+ } catch (Throwable t) {
+ failure = t;
+ } finally {
+ failure = closeEngine(failure, engine);
+ failure = closeManager(failure, manager);
+ failure = freePayload(failure, payload);
+ failure = removeRoot(failure, root);
+ }
+ rethrow(failure);
+ }
+ });
+ }
+
+ @Test
+ public void testFinalWatermarkBarrierFailureRetainsSlotUntilRetry() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ for (String barrier : Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync")) {
+ String root = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-close-barrier-" + barrier + "-" + System.nanoTime()).toString();
+ String slot = root + "/slot";
+ SegmentManager manager = null;
+ CursorSendEngine predecessor = null;
+ CursorSendEngine successor = null;
+ CrashImageFilesFacade ff = null;
+ long payload = 0;
+ Throwable failure = null;
+ try {
+ Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT));
+ ff = new CrashImageFilesFacade(slot, -1);
+ long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L;
+ manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60),
+ SegmentManager.UNLIMITED_TOTAL_BYTES, ff);
+ payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT);
+ predecessor = new CursorSendEngine(slot, segmentSize, manager);
+ Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7);
+ Assert.assertEquals(0L, predecessor.appendBlocking(payload, 32));
+ Assert.assertTrue(predecessor.acknowledge(0L));
+
+ ff.failPersistently(barrier);
+ try {
+ predecessor.close();
+ Assert.fail("expected close to report failed final barrier " + barrier);
+ } catch (IllegalStateException expected) {
+ Assert.assertTrue("unexpected close failure: " + expected,
+ expected.getMessage().contains("ack watermark")
+ || expected.getMessage().contains("slot directory"));
+ }
+ try {
+ successor = new CursorSendEngine(slot, segmentSize, manager);
+ Assert.fail("successor acquired a slot whose final ACK barrier failed: " + barrier);
+ } catch (IllegalStateException expected) {
+ Assert.assertTrue("successor failed for the wrong reason: " + expected,
+ expected.getMessage().contains("slot already in use"));
+ }
+
+ ff.clearPersistentFailure();
+ awaitCloseCompleted(predecessor);
+ predecessor = null;
+
+ successor = new CursorSendEngine(slot, segmentSize, manager);
+ Assert.assertEquals("successful retry must leave no ACKed frame to replay",
+ -1L, successor.publishedFsn());
+ successor.close();
+ Assert.assertTrue(successor.isCloseCompleted());
+ successor = null;
+ } catch (Throwable t) {
+ failure = t;
+ } finally {
+ if (failure != null && ff != null) {
+ // Let a retained predecessor converge before cleanup.
+ // The exact red assertion remains attached to failure.
+ ff.clearPersistentFailure();
+ }
+ failure = closeEngine(failure, successor);
+ failure = closeEngine(failure, predecessor);
+ failure = closeManager(failure, manager);
+ failure = freePayload(failure, payload);
+ failure = removeRoot(failure, root);
+ }
+ rethrow(failure);
+ }
+ });
+ }
+
+ @Test
+ public void testRecoveredSlotRejectsMissingWatermark() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ String root = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-missing-watermark-" + System.nanoTime()).toString();
+ String slot = root + "/slot";
+ SegmentManager manager = null;
+ CursorSendEngine predecessor = null;
+ CursorSendEngine successor = null;
+ long payload = 0;
+ Throwable failure = null;
+ try {
+ Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT));
+ CrashImageFilesFacade ff = new CrashImageFilesFacade(slot, -1);
+ long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L;
+ manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60),
+ SegmentManager.UNLIMITED_TOTAL_BYTES, ff);
+ payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT);
+ predecessor = new CursorSendEngine(slot, segmentSize, manager);
+ Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7);
+ Assert.assertEquals(0L, predecessor.appendBlocking(payload, 32));
+ Assert.assertTrue(predecessor.acknowledge(0L));
+ ff.beginClose();
+ ff.blockSegmentRemove = true;
+ predecessor.close();
+ Assert.assertTrue(predecessor.isCloseCompleted());
+ predecessor = null;
+ Assert.assertTrue("precondition: ACKed segment residue must survive",
+ Files.exists(slot + "/sf-initial.sfa"));
+
+ ff.failWatermarkOpen = true;
+ try {
+ successor = new CursorSendEngine(slot, segmentSize, manager);
+ Assert.fail("recovered disk slot started without a usable ACK watermark");
+ } catch (IllegalStateException expected) {
+ Assert.assertTrue("successor failed for the wrong reason: " + expected,
+ expected.getMessage().contains("ack watermark"));
+ }
+
+ ff.failWatermarkOpen = false;
+ ff.blockSegmentRemove = false;
+ successor = new CursorSendEngine(slot, segmentSize, manager);
+ Assert.assertTrue(successor.wasRecoveredFromDisk());
+ Assert.assertTrue("successor exposes predecessor's ACKed frame",
+ successor.ackedFsn() >= successor.publishedFsn());
+ successor.close();
+ Assert.assertTrue(successor.isCloseCompleted());
+ successor = null;
+ } catch (Throwable t) {
+ failure = t;
+ } finally {
+ failure = closeEngine(failure, successor);
+ failure = closeEngine(failure, predecessor);
+ failure = closeManager(failure, manager);
+ failure = freePayload(failure, payload);
+ failure = removeRoot(failure, root);
+ }
+ rethrow(failure);
+ });
+ }
+
+ private static Throwable addCleanupFailure(Throwable failure, Throwable cleanupFailure) {
+ if (failure == null) {
+ return cleanupFailure;
+ }
+ if (failure != cleanupFailure) {
+ failure.addSuppressed(cleanupFailure);
+ }
+ return failure;
+ }
+
+ private static void awaitCloseCompleted(CursorSendEngine engine) {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (!engine.isCloseCompleted() && System.nanoTime() < deadline) {
+ Thread.yield();
+ }
+ Assert.assertTrue("close retry did not complete", engine.isCloseCompleted());
+ }
+
+ private static Throwable closeEngine(Throwable failure, CursorSendEngine engine) {
+ if (engine != null) {
+ try {
+ engine.close();
+ } catch (Throwable cleanupFailure) {
+ failure = addCleanupFailure(failure, cleanupFailure);
+ }
+ }
+ return failure;
+ }
+
+ private static Throwable closeManager(Throwable failure, SegmentManager manager) {
+ if (manager != null) {
+ try {
+ manager.close();
+ } catch (Throwable cleanupFailure) {
+ failure = addCleanupFailure(failure, cleanupFailure);
+ }
+ }
+ return failure;
+ }
+
+ private static Throwable freePayload(Throwable failure, long payload) {
+ if (payload != 0) {
+ try {
+ Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT);
+ } catch (Throwable cleanupFailure) {
+ failure = addCleanupFailure(failure, cleanupFailure);
+ }
+ }
+ return failure;
+ }
+
+ private static void removeRecursive(String dir) {
+ long find = Files.findFirst(dir);
+ if (find > 0) {
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ if (name != null && !".".equals(name) && !"..".equals(name)) {
+ String child = dir + "/" + name;
+ if (!Files.remove(child)) {
+ removeRecursive(child);
+ }
+ }
+ rc = Files.findNext(find);
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ }
+ Files.remove(dir);
+ }
+
+ private static Throwable removeRoot(Throwable failure, String root) {
+ try {
+ removeRecursive(root);
+ Assert.assertFalse("test directory was not removed: " + root, Files.exists(root));
+ } catch (Throwable cleanupFailure) {
+ failure = addCleanupFailure(failure, cleanupFailure);
+ }
+ return failure;
+ }
+
+ private static void rethrow(Throwable failure) {
+ if (failure != null) {
+ CursorSendEngineCrashConsistencyTest.throwUnchecked(failure);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void throwUnchecked(Throwable failure) throws T {
+ throw (T) failure;
+ }
+
+ private static final class CrashImageFilesFacade implements FilesFacade {
+ private final List events = new ArrayList<>();
+ private final int failAt;
+ private final String slot;
+ private boolean active;
+ private boolean blockSegmentRemove;
+ private boolean durableSegments = true;
+ private boolean durableWatermark;
+ private int eventIndex;
+ private boolean failWatermarkOpen;
+ private String persistentFailure;
+ private int watermarkFd = -1;
+
+ private CrashImageFilesFacade(String slot, int failAt) {
+ this.slot = slot;
+ this.failAt = failAt;
+ }
+
+ private void beginClose() {
+ active = true;
+ events.clear();
+ eventIndex = 0;
+ }
+
+ private void clearPersistentFailure() {
+ persistentFailure = null;
+ }
+
+ private boolean fail(String event) {
+ events.add(event);
+ return event.equals(persistentFailure) || eventIndex++ == failAt;
+ }
+
+ private void failPersistently(String event) {
+ active = true;
+ events.clear();
+ eventIndex = 0;
+ persistentFailure = event;
+ }
+
+ @Override
+ public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); }
+ @Override
+ public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); }
+ @Override
+ public int close(int fd) { return INSTANCE.close(fd); }
+ @Override
+ public boolean exists(String path) { return INSTANCE.exists(path); }
+ @Override
+ public void findClose(long findPtr) { INSTANCE.findClose(findPtr); }
+ @Override
+ public long findFirst(String dir) { return INSTANCE.findFirst(dir); }
+ @Override
+ public long findName(long findPtr) { return INSTANCE.findName(findPtr); }
+ @Override
+ public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); }
+ @Override
+ public int findType(long findPtr) { return INSTANCE.findType(findPtr); }
+ @Override
+ public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); }
+ @Override
+ public int fsync(int fd) {
+ return active && fd == watermarkFd && fail("watermark-fsync") ? -1 : INSTANCE.fsync(fd);
+ }
+ @Override
+ public int fsyncDir(String dir) {
+ if (active && slot.equals(dir)) {
+ if (fail("dir-fsync")) return -1;
+ if (events.contains("segment-remove")) {
+ durableSegments = false;
+ } else {
+ durableWatermark = true;
+ }
+ }
+ return INSTANCE.fsyncDir(dir);
+ }
+ @Override
+ public long length(int fd) { return INSTANCE.length(fd); }
+ @Override
+ public long length(String path) { return INSTANCE.length(path); }
+ @Override
+ public long length(long pathPtr) { return INSTANCE.length(pathPtr); }
+ @Override
+ public int lock(int fd) { return INSTANCE.lock(fd); }
+ @Override
+ public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); }
+ @Override
+ public int msync(long addr, long len, boolean async) {
+ return active && fail("watermark-msync") ? -1 : INSTANCE.msync(addr, len, async);
+ }
+ @Override
+ public int openCleanRW(String path) {
+ if (failWatermarkOpen && path.equals(slot + "/" + AckWatermark.FILE_NAME)) return -1;
+ int fd = INSTANCE.openCleanRW(path);
+ if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd;
+ return fd;
+ }
+ @Override
+ public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); }
+ @Override
+ public int openRW(String path) {
+ if (failWatermarkOpen && path.equals(slot + "/" + AckWatermark.FILE_NAME)) return -1;
+ int fd = INSTANCE.openRW(path);
+ if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd;
+ return fd;
+ }
+ @Override
+ public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); }
+ @Override
+ public long read(int fd, long addr, long len, long offset) {
+ return INSTANCE.read(fd, addr, len, offset);
+ }
+ @Override
+ public boolean remove(String path) {
+ if (active && path.endsWith(".sfa")) {
+ if (blockSegmentRemove || fail("segment-remove")) return false;
+ } else if (active && path.equals(slot + "/" + AckWatermark.FILE_NAME)) {
+ if (fail("watermark-remove")) return false;
+ }
+ return INSTANCE.remove(path);
+ }
+ @Override
+ public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); }
+ @Override
+ public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); }
+ @Override
+ public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); }
+ @Override
+ public long write(int fd, long addr, long len, long offset) {
+ return INSTANCE.write(fd, addr, len, offset);
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java
new file mode 100644
index 00000000..2c5f76c3
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java
@@ -0,0 +1,865 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.nio.file.Paths;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Engine-level regression for the shutdown hazard where
+ * {@link CursorSendEngine#close()} released the slot lock, closed the ring
+ * and watermark, and unlinked segment files while the shared
+ * {@link SegmentManager} worker was still mid service pass for the engine's
+ * ring. A replacement engine could acquire the same slot the moment the
+ * lock was released, after which the stale worker's abandon/trim path could
+ * unlink a segment path the replacement was actively writing through —
+ * store-and-forward data loss after restart.
+ *
+ * The fix makes {@code close()} run a quiescence barrier
+ * ({@link SegmentManager#awaitRingQuiescence}) after {@code deregister} and
+ * refuse to release any worker-reachable resource (ring, watermark, segment
+ * files, slot lock) until the barrier confirms the worker cannot touch the
+ * slot again. On barrier timeout an owned-manager engine hands cleanup
+ * ownership to the worker's exit path (see
+ * {@code SegmentManager.deferUntilWorkerExit}); a shared-manager engine
+ * deliberately leaks and a later {@code close()} retries the cleanup.
+ */
+public class CursorSendEngineSlotReacquisitionTest {
+
+ private String tmpDir;
+
+ @Before
+ public void setUp() {
+ tmpDir = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-engine-slot-reacq-" + System.nanoTime()).toString();
+ Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT));
+ }
+
+ @After
+ public void tearDown() {
+ if (tmpDir == null) return;
+ rmDirRecursive(tmpDir);
+ Files.remove(tmpDir);
+ }
+
+ /**
+ * The structural guarantee: while the manager worker is provably still
+ * inside a service pass for the engine's ring, repeated direct
+ * {@code close()} calls must NOT hand the slot to anyone else. With the
+ * duplicate-cleanup-owner branch reverted, the second close runs cleanup
+ * inline and the mid-test {@code SlotLock.acquire} probe succeeds. Once
+ * the pass finishes, its deferred cleanup must run exactly once and
+ * release the slot without another close retry.
+ */
+ @Test(timeout = 30_000L)
+ public void testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32);
+ String slot = tmpDir + "/slot";
+ // 60 s poll: the worker only acts when explicitly woken, so the
+ // single pass we park below is the only pass in flight.
+ SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60));
+ CountDownLatch cleanupFinished = new CountDownLatch(1);
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean fired = new AtomicBoolean();
+ AtomicInteger cleanupCount = new AtomicInteger();
+ AtomicReference hookErr = new AtomicReference<>();
+ boolean managerClosed = false;
+ CursorSendEngine engine = null;
+ try {
+ manager.setAfterRingCleanupHook(() -> {
+ cleanupCount.incrementAndGet();
+ cleanupFinished.countDown();
+ });
+ manager.setBeforeInstallSyncHook(() -> {
+ if (!fired.compareAndSet(false, true)) return;
+ workerBlocked.countDown();
+ try {
+ if (!releaseWorker.await(20, TimeUnit.SECONDS)) {
+ hookErr.compareAndSet(null,
+ new AssertionError("timed out waiting for test to release worker"));
+ }
+ } catch (Throwable t) {
+ hookErr.compareAndSet(null, t);
+ }
+ });
+ manager.start();
+
+ // Shared manager: ownsManager=false, so engine close() cannot
+ // fall back on manager.close()'s join — the per-ring barrier
+ // is the only protection, which is exactly what we pin here.
+ engine = new CursorSendEngine(slot, segSize, manager);
+ Assert.assertTrue("worker never reached the install hook",
+ workerBlocked.await(5, TimeUnit.SECONDS));
+
+ // Barrier must time out fast: the worker is parked inside the
+ // service pass for this engine's ring.
+ manager.setWorkerJoinTimeoutMillis(50L);
+ engine.close();
+ Assert.assertFalse("incomplete close must remain observable to the owner",
+ engine.isCloseCompleted());
+
+ // Exercise CursorSendEngine.close() directly again while the
+ // same pass already owns deferredClose. Sender.close() cannot
+ // reach this branch because its second call is a no-op.
+ engine.close();
+ Assert.assertFalse("repeated close must not steal deferred cleanup ownership",
+ engine.isCloseCompleted());
+ Assert.assertEquals("cleanup ran while the worker pass was blocked",
+ 0, cleanupCount.get());
+
+ // The slot must still be locked: a replacement engine (or raw
+ // SlotLock) acquiring it now would race the stale worker.
+ try {
+ SlotLock probe = SlotLock.acquire(slot);
+ probe.close();
+ Assert.fail("engine.close() released the slot lock while the manager "
+ + "worker was still mid service pass for its ring — a "
+ + "replacement engine could acquire the slot and have its "
+ + "segment files unlinked by the stale worker");
+ } catch (Exception expected) {
+ // good — slot retained.
+ }
+
+ // Let the worker finish its pass (it abandons the spare: the
+ // ring was deregistered by the close attempt above).
+ releaseWorker.countDown();
+ Assert.assertTrue("ring pass did not finish deferred cleanup",
+ cleanupFinished.await(5, TimeUnit.SECONDS));
+ Assert.assertEquals("ring-pass cleanup must run exactly once",
+ 1, cleanupCount.get());
+ Assert.assertTrue("ring-pass cleanup must report complete cleanup",
+ engine.isCloseCompleted());
+ engine = null;
+
+ try (SlotLock probe = SlotLock.acquire(slot)) {
+ Assert.assertNotNull("slot must be acquirable after a completed close", probe);
+ } catch (Exception e) {
+ throw new AssertionError("ring-pass cleanup did not release the slot lock", e);
+ }
+
+ manager.close();
+ managerClosed = true;
+ if (hookErr.get() != null) {
+ throw new AssertionError("install hook failed", hookErr.get());
+ }
+ } finally {
+ manager.setAfterRingCleanupHook(null);
+ manager.setBeforeInstallSyncHook(null);
+ releaseWorker.countDown();
+ if (!managerClosed) {
+ manager.close();
+ }
+ }
+ });
+ }
+
+ /**
+ * Owned-manager twin of {@link #testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass}:
+ * the ONLY construction shape production uses (Sender.build, BackgroundDrainer,
+ * QwpWebSocketSender.connect all own their manager). The owned close path does
+ * not run the per-ring barrier at all — it relies on {@code manager.close()}'s
+ * bounded join and the {@code isWorkerReaped()} check. If that check regressed
+ * to report quiescence unconditionally (or {@code isWorkerReaped()} itself
+ * returned true while the worker is alive), close() would release the slot
+ * lock mid service pass and the shared-manager tests would stay green — this
+ * test is the red gate for the production path.
+ *
+ * Determinism: the owned manager starts inside the engine ctor (1 ms poll),
+ * so its first spare-install pass races test setup. We first wait until the
+ * initial hot spare is installed — after that the worker cannot enter another
+ * install pass until a rotation consumes the spare, so the park hook installed
+ * afterwards can neither be missed nor fire early. Two appends then fill the
+ * active segment and rotate onto the spare; the worker's next poll tick
+ * re-enters the install pass and parks in the hook.
+ */
+ @Test(timeout = 30_000L)
+ public void testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ final int payloadLen = 32;
+ long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen);
+ String slot = tmpDir + "/owned-parked-slot";
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean fired = new AtomicBoolean();
+ AtomicReference hookErr = new AtomicReference<>();
+ // Production shape: private, owned manager (ownsManager=true).
+ CursorSendEngine engine = new CursorSendEngine(slot, segSize);
+ SegmentManager manager = readManager(engine);
+ long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
+ try {
+ // Phase 1: let the worker finish the initial spare install so
+ // the hook below can only fire on the rotation-triggered pass.
+ SegmentRing ring = readRing(engine);
+ long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (ring.needsHotSpare()) {
+ if (System.nanoTime() > deadlineNs) {
+ throw new AssertionError("manager worker never installed the initial hot spare");
+ }
+ Thread.sleep(1);
+ }
+ manager.setBeforeInstallSyncHook(() -> {
+ if (!fired.compareAndSet(false, true)) return;
+ workerBlocked.countDown();
+ try {
+ if (!releaseWorker.await(20, TimeUnit.SECONDS)) {
+ hookErr.compareAndSet(null,
+ new AssertionError("timed out waiting for test to release worker"));
+ }
+ } catch (Throwable t) {
+ hookErr.compareAndSet(null, t);
+ }
+ });
+
+ // Phase 2: one frame fills the active segment exactly; the
+ // second forces rotation onto the spare. needsHotSpare() is
+ // true again, so the worker's next tick parks in the hook.
+ Unsafe.getUnsafe().putLong(buf, 0L);
+ Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertTrue("worker never re-entered a spare-install pass",
+ workerBlocked.await(5, TimeUnit.SECONDS));
+
+ // Phase 3: owned close with the worker provably mid service
+ // pass. manager.close()'s 50 ms join times out, the worker is
+ // not reaped, and close() must retain every worker-reachable
+ // resource — above all the slot flock.
+ manager.setWorkerJoinTimeoutMillis(50L);
+ engine.close();
+ Assert.assertFalse("incomplete owned close must remain observable to the owner",
+ engine.isCloseCompleted());
+ try {
+ SlotLock probe = SlotLock.acquire(slot);
+ probe.close();
+ Assert.fail("owned engine.close() released the slot lock while its manager "
+ + "worker was still mid service pass — a replacement engine could "
+ + "acquire the slot and have its segment files unlinked by the "
+ + "stale worker (the production SF-data-loss hazard)");
+ } catch (Exception expected) {
+ // good — slot retained.
+ }
+
+ // Phase 4: release the worker (it abandons the spare: the ring
+ // was deregistered by the close attempt) and retry. The join
+ // now reaps the worker and the full cleanup must complete.
+ releaseWorker.countDown();
+ manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60));
+ engine.close();
+ Assert.assertTrue("retried owned close must report complete cleanup",
+ engine.isCloseCompleted());
+ try (SlotLock probe = SlotLock.acquire(slot)) {
+ Assert.assertNotNull("slot must be acquirable after a completed close", probe);
+ } catch (Exception e) {
+ throw new AssertionError("retried owned close() did not release the slot lock", e);
+ }
+ if (hookErr.get() != null) {
+ throw new AssertionError("install hook failed", hookErr.get());
+ }
+ } finally {
+ Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT);
+ manager.setBeforeInstallSyncHook(null);
+ releaseWorker.countDown();
+ manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60));
+ try {
+ engine.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ });
+ }
+
+ /**
+ * The ownership handoff (owned manager): when close() cannot confirm
+ * worker quiescence within the bounded join, the terminal cleanup (ring,
+ * watermark, flock release) transfers to the worker's exit path — the
+ * worker is provably the last thread able to touch the slot directory.
+ * Once the parked pass is released the worker must run the cleanup
+ * itself, WITHOUT any retried {@code close()}: {@code isCloseCompleted()}
+ * flips true and the slot becomes acquirable again. This is what lets a
+ * pool recover a retired slot instead of losing its capacity until
+ * process exit.
+ */
+ @Test(timeout = 30_000L)
+ public void testOwnedEngineCloseHandsCleanupToWorkerExit() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ final int payloadLen = 32;
+ long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen);
+ String slot = tmpDir + "/owned-handoff-slot";
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean fired = new AtomicBoolean();
+ AtomicReference hookErr = new AtomicReference<>();
+ // Production shape: private, owned manager (ownsManager=true).
+ CursorSendEngine engine = new CursorSendEngine(slot, segSize);
+ SegmentManager manager = readManager(engine);
+ long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
+ try {
+ // Phase 1: wait out the initial spare install so the park hook
+ // can only fire on the rotation-triggered pass (see
+ // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass).
+ SegmentRing ring = readRing(engine);
+ long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (ring.needsHotSpare()) {
+ if (System.nanoTime() > deadlineNs) {
+ throw new AssertionError("manager worker never installed the initial hot spare");
+ }
+ Thread.sleep(1);
+ }
+ manager.setBeforeInstallSyncHook(() -> {
+ if (!fired.compareAndSet(false, true)) return;
+ workerBlocked.countDown();
+ try {
+ if (!releaseWorker.await(20, TimeUnit.SECONDS)) {
+ hookErr.compareAndSet(null,
+ new AssertionError("timed out waiting for test to release worker"));
+ }
+ } catch (Throwable t) {
+ hookErr.compareAndSet(null, t);
+ }
+ });
+
+ // Phase 2: fill the active segment and rotate onto the spare;
+ // the worker's next tick re-enters the install pass and parks.
+ Unsafe.getUnsafe().putLong(buf, 0L);
+ Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertTrue("worker never re-entered a spare-install pass",
+ workerBlocked.await(5, TimeUnit.SECONDS));
+
+ // Phase 3: owned close with the worker provably mid-pass. The
+ // bounded join times out; cleanup ownership is handed to the
+ // worker's exit path. Every worker-reachable resource — above
+ // all the slot flock — must still be retained at this point.
+ manager.setWorkerJoinTimeoutMillis(50L);
+ engine.close();
+ Assert.assertFalse("close must stay incomplete while the worker holds the handoff",
+ engine.isCloseCompleted());
+ try {
+ SlotLock probe = SlotLock.acquire(slot);
+ probe.close();
+ Assert.fail("engine.close() released the slot lock while its manager worker "
+ + "was still mid service pass — the handoff must not weaken the "
+ + "quiescence gate");
+ } catch (Exception expected) {
+ // good — slot retained while the worker can still touch it.
+ }
+
+ // Phase 4 — the contract under test: release the worker and do
+ // NOT retry close(). The worker finishes its pass, exits, and
+ // runs the deferred cleanup itself, flipping isCloseCompleted
+ // and releasing the flock with no further caller action.
+ releaseWorker.countDown();
+ long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (!engine.isCloseCompleted()) {
+ if (System.nanoTime() > cleanupDeadlineNs) {
+ throw new AssertionError(
+ "deferred cleanup never ran on manager-worker exit — the slot "
+ + "would stay retired until process exit");
+ }
+ Thread.sleep(1);
+ }
+ try (SlotLock probe = SlotLock.acquire(slot)) {
+ Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe);
+ } catch (Exception e) {
+ throw new AssertionError("worker-exit cleanup did not release the slot lock", e);
+ }
+ if (hookErr.get() != null) {
+ throw new AssertionError("install hook failed", hookErr.get());
+ }
+ } finally {
+ Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT);
+ manager.setBeforeInstallSyncHook(null);
+ releaseWorker.countDown();
+ manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60));
+ try {
+ engine.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ });
+ }
+
+ /**
+ * Registration-failure twin of
+ * {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}: when
+ * {@code deferUntilWorkerExit} itself throws (allocation failure while
+ * building the handoff), close() must NOT mistake the swallowed throw
+ * for "worker already exited" and run the terminal cleanup inline — the
+ * worker is provably still mid service pass, so releasing the ring,
+ * watermark or slot flock here is the original stale-worker UAF/data-loss
+ * hazard. Every worker-reachable resource must be retained and the close
+ * must stay incomplete; a retried close() after the worker exits
+ * converges and releases the slot.
+ */
+ @Test(timeout = 30_000L)
+ public void testOwnedEngineCloseRetainsSlotWhenHandoffRegistrationFails() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ final int payloadLen = 32;
+ long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen);
+ String slot = tmpDir + "/owned-regfail-slot";
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean fired = new AtomicBoolean();
+ AtomicReference hookErr = new AtomicReference<>();
+ // Production shape: private, owned manager (ownsManager=true).
+ CursorSendEngine engine = new CursorSendEngine(slot, segSize);
+ SegmentManager manager = readManager(engine);
+ long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
+ try {
+ // Phase 1: wait out the initial spare install so the park hook
+ // can only fire on the rotation-triggered pass (see
+ // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass).
+ SegmentRing ring = readRing(engine);
+ long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (ring.needsHotSpare()) {
+ if (System.nanoTime() > deadlineNs) {
+ throw new AssertionError("manager worker never installed the initial hot spare");
+ }
+ Thread.sleep(1);
+ }
+ manager.setBeforeInstallSyncHook(() -> {
+ if (!fired.compareAndSet(false, true)) return;
+ workerBlocked.countDown();
+ try {
+ if (!releaseWorker.await(20, TimeUnit.SECONDS)) {
+ hookErr.compareAndSet(null,
+ new AssertionError("timed out waiting for test to release worker"));
+ }
+ } catch (Throwable t) {
+ hookErr.compareAndSet(null, t);
+ }
+ });
+
+ // Phase 2: fill the active segment and rotate onto the spare;
+ // the worker's next tick re-enters the install pass and parks.
+ Unsafe.getUnsafe().putLong(buf, 0L);
+ Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertTrue("worker never re-entered a spare-install pass",
+ workerBlocked.await(5, TimeUnit.SECONDS));
+
+ // Phase 3: make the handoff registration throw — simulating
+ // an OutOfMemoryError allocating the cleanup lambda/list —
+ // with the worker provably still mid service pass. close()
+ // must retain everything: no inline finishClose, no flock
+ // release, closeCompleted stays false.
+ manager.setBeforeExitCleanupRegistrationHook(() -> {
+ throw new OutOfMemoryError("simulated allocation failure registering exit cleanup");
+ });
+ manager.setWorkerJoinTimeoutMillis(50L);
+ engine.close();
+ Assert.assertFalse("close must stay incomplete when handoff registration fails",
+ engine.isCloseCompleted());
+ try {
+ SlotLock probe = SlotLock.acquire(slot);
+ probe.close();
+ Assert.fail("engine.close() released the slot lock after a failed handoff "
+ + "registration while the manager worker was still mid service "
+ + "pass — the swallowed throw was mistaken for proof the worker "
+ + "exited (stale-worker UAF/data-loss hazard)");
+ } catch (Exception expected) {
+ // good — slot retained while the worker can still touch it.
+ }
+
+ // Phase 4: clear the fault, release the worker (its loop was
+ // already stopped by the close attempt, so it exits), and
+ // retry close(). The retry must converge via isWorkerReaped()
+ // and release the slot.
+ manager.setBeforeExitCleanupRegistrationHook(null);
+ releaseWorker.countDown();
+ manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60));
+ engine.close();
+ Assert.assertTrue("retried close must report complete cleanup",
+ engine.isCloseCompleted());
+ try (SlotLock probe = SlotLock.acquire(slot)) {
+ Assert.assertNotNull("slot must be acquirable after the retried close", probe);
+ } catch (Exception e) {
+ throw new AssertionError("retried close() did not release the slot lock", e);
+ }
+ if (hookErr.get() != null) {
+ throw new AssertionError("install hook failed", hookErr.get());
+ }
+ } finally {
+ Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT);
+ manager.setBeforeInstallSyncHook(null);
+ manager.setBeforeExitCleanupRegistrationHook(null);
+ releaseWorker.countDown();
+ manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60));
+ try {
+ engine.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ });
+ }
+
+ /**
+ * Exactly-once contention on the terminal-cleanup claim
+ * ({@code terminalCleanupClaimed} CAS): after a timed-out owned close
+ * handed cleanup to the worker's exit path, a retried {@code close()}
+ * that races the worker MID-{@code finishClose} must neither re-run the
+ * terminal cleanup (double munmap / double flock release) nor block on
+ * the worker, nor publish completion on the worker's behalf. The race
+ * window is made deterministic by parking the worker inside
+ * {@code finishClose} (via {@code beforeFlockReleaseHook}) while the
+ * retried close() converges through {@code isWorkerReaped()} and loses
+ * the CAS.
+ */
+ @Test(timeout = 30_000L)
+ public void testTerminalCleanupRunsExactlyOnceWhenRetriedCloseRacesWorkerHandoff() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ final int payloadLen = 32;
+ long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen);
+ String slot = tmpDir + "/cas-contention-slot";
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ CountDownLatch inFinishClose = new CountDownLatch(1);
+ CountDownLatch releaseFinishClose = new CountDownLatch(1);
+ AtomicBoolean fired = new AtomicBoolean();
+ AtomicInteger finishCloseRuns = new AtomicInteger();
+ AtomicReference hookErr = new AtomicReference<>();
+ // Production shape: private, owned manager (ownsManager=true).
+ CursorSendEngine engine = new CursorSendEngine(slot, segSize);
+ SegmentManager manager = readManager(engine);
+ long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
+ try {
+ // Phase 1: wait out the initial spare install so the park hook
+ // can only fire on the rotation-triggered pass.
+ SegmentRing ring = readRing(engine);
+ long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (ring.needsHotSpare()) {
+ if (System.nanoTime() > deadlineNs) {
+ throw new AssertionError("manager worker never installed the initial hot spare");
+ }
+ Thread.sleep(1);
+ }
+ manager.setBeforeInstallSyncHook(() -> {
+ if (!fired.compareAndSet(false, true)) return;
+ workerBlocked.countDown();
+ try {
+ if (!releaseWorker.await(20, TimeUnit.SECONDS)) {
+ hookErr.compareAndSet(null,
+ new AssertionError("timed out waiting for test to release worker"));
+ }
+ } catch (Throwable t) {
+ hookErr.compareAndSet(null, t);
+ }
+ });
+ // Counts terminal-cleanup executions and parks the FIRST one
+ // (the worker's deferred cleanup) mid-finishClose, before the
+ // flock release — the exact window a retried close() races.
+ engine.setBeforeFlockReleaseHook(() -> {
+ if (finishCloseRuns.incrementAndGet() == 1) {
+ inFinishClose.countDown();
+ try {
+ if (!releaseFinishClose.await(20, TimeUnit.SECONDS)) {
+ hookErr.compareAndSet(null, new AssertionError(
+ "timed out waiting for test to release finishClose"));
+ }
+ } catch (Throwable t) {
+ hookErr.compareAndSet(null, t);
+ }
+ }
+ });
+
+ // Phase 2: rotate onto the spare so the worker parks in the
+ // next install pass.
+ Unsafe.getUnsafe().putLong(buf, 0L);
+ Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertTrue("worker never re-entered a spare-install pass",
+ workerBlocked.await(5, TimeUnit.SECONDS));
+
+ // Phase 3: timed-out close — cleanup ownership transfers to
+ // the worker's exit path.
+ manager.setWorkerJoinTimeoutMillis(50L);
+ engine.close();
+ Assert.assertFalse("close must stay incomplete while the worker holds the handoff",
+ engine.isCloseCompleted());
+
+ // Phase 4: release the pass. The worker exits its loop, wins
+ // the cleanup CAS, enters finishClose and parks in the hook —
+ // mid-cleanup, flock still held, completion unpublished.
+ releaseWorker.countDown();
+ Assert.assertTrue("worker never entered the deferred finishClose",
+ inFinishClose.await(10, TimeUnit.SECONDS));
+ Assert.assertEquals(1, finishCloseRuns.get());
+ Assert.assertFalse("completion must not be observable mid-finishClose",
+ engine.isCloseCompleted());
+
+ // Phase 5 — the contention under test: a retried close() while
+ // the worker is parked INSIDE finishClose. The worker loop has
+ // already exited (workerLoopExited=true precedes the deferred
+ // cleanups), so the short bounded join reaps the manager state
+ // and close() converges to the CAS — which it must LOSE,
+ // returning promptly without touching ring/watermark/flock.
+ engine.close();
+ Assert.assertEquals(
+ "retried close() re-ran the terminal cleanup while the worker's "
+ + "deferred cleanup was mid-flight — ring/watermark/flock would "
+ + "be double-released",
+ 1, finishCloseRuns.get());
+ Assert.assertFalse("retried close() must not publish completion on the worker's behalf",
+ engine.isCloseCompleted());
+ try {
+ SlotLock probe = SlotLock.acquire(slot);
+ probe.close();
+ Assert.fail("slot lock observable as released while the worker was still "
+ + "parked before its flock release");
+ } catch (Exception expected) {
+ // good — flock still held by the parked cleanup.
+ }
+
+ // Phase 6: let the worker finish. Completion publishes, the
+ // slot becomes acquirable, and the cleanup count stays at 1.
+ releaseFinishClose.countDown();
+ long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (!engine.isCloseCompleted()) {
+ if (System.nanoTime() > cleanupDeadlineNs) {
+ throw new AssertionError("deferred cleanup never completed after release");
+ }
+ Thread.sleep(1);
+ }
+ Assert.assertEquals(1, finishCloseRuns.get());
+ try (SlotLock probe = SlotLock.acquire(slot)) {
+ Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe);
+ }
+ // A final close() takes the fast no-op path.
+ engine.close();
+ Assert.assertEquals("post-completion close() must be a no-op",
+ 1, finishCloseRuns.get());
+ if (hookErr.get() != null) {
+ throw new AssertionError("hook failed", hookErr.get());
+ }
+ } finally {
+ Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT);
+ manager.setBeforeInstallSyncHook(null);
+ engine.setBeforeFlockReleaseHook(null);
+ releaseWorker.countDown();
+ releaseFinishClose.countDown();
+ manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60));
+ try {
+ engine.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ });
+ }
+
+ /**
+ * Memory-mode twin of {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}:
+ * {@code sfDir == null}, so there is no slot lock, no watermark and no
+ * segment files — but the ring's malloc'd native segments are still
+ * worker-reachable, so the timed-out close must take the same handoff
+ * path with every SF-only resource null. Pins that (a) the handoff
+ * branch tolerates null slotLock/watermark/sfDir without NPE, (b) the
+ * close stays incomplete while the worker can still touch the ring, and
+ * (c) the worker-exit cleanup completes the close and frees the ring's
+ * native memory (assertMemoryLeak is the leak oracle here).
+ */
+ @Test(timeout = 30_000L)
+ public void testMemoryModeOwnedCloseHandsCleanupToWorkerExit() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ final int payloadLen = 32;
+ long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen);
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean fired = new AtomicBoolean();
+ AtomicReference hookErr = new AtomicReference<>();
+ // Memory mode: null sfDir, private owned manager — the exact
+ // shape non-SF async ingest uses.
+ CursorSendEngine engine = new CursorSendEngine(null, segSize);
+ SegmentManager manager = readManager(engine);
+ long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
+ try {
+ // Phase 1: wait out the initial spare install so the park hook
+ // can only fire on the rotation-triggered pass.
+ SegmentRing ring = readRing(engine);
+ long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (ring.needsHotSpare()) {
+ if (System.nanoTime() > deadlineNs) {
+ throw new AssertionError("manager worker never installed the initial hot spare");
+ }
+ Thread.sleep(1);
+ }
+ manager.setBeforeInstallSyncHook(() -> {
+ if (!fired.compareAndSet(false, true)) return;
+ workerBlocked.countDown();
+ try {
+ if (!releaseWorker.await(20, TimeUnit.SECONDS)) {
+ hookErr.compareAndSet(null,
+ new AssertionError("timed out waiting for test to release worker"));
+ }
+ } catch (Throwable t) {
+ hookErr.compareAndSet(null, t);
+ }
+ });
+
+ // Phase 2: rotate onto the spare; the worker's next tick
+ // re-enters the (in-memory) install pass and parks.
+ Unsafe.getUnsafe().putLong(buf, 0L);
+ Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen));
+ Assert.assertTrue("worker never re-entered a spare-install pass",
+ workerBlocked.await(5, TimeUnit.SECONDS));
+
+ // Phase 3: timed-out memory-mode close. The worker can still
+ // touch the ring's native memory, so the close must hand off
+ // and stay incomplete — releasing the ring here would be a
+ // use-after-free on the worker's install path.
+ manager.setWorkerJoinTimeoutMillis(50L);
+ engine.close();
+ Assert.assertFalse(
+ "memory-mode close must stay incomplete while the worker is mid service pass",
+ engine.isCloseCompleted());
+
+ // Phase 4: release the worker; its exit path must run the
+ // deferred cleanup (null slotLock/watermark/sfDir) and flip
+ // completion with no further caller action.
+ releaseWorker.countDown();
+ long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (!engine.isCloseCompleted()) {
+ if (System.nanoTime() > cleanupDeadlineNs) {
+ throw new AssertionError(
+ "deferred memory-mode cleanup never ran on manager-worker exit — "
+ + "the ring's native segments would leak for the process lifetime");
+ }
+ Thread.sleep(1);
+ }
+ if (hookErr.get() != null) {
+ throw new AssertionError("install hook failed", hookErr.get());
+ }
+ } finally {
+ Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT);
+ manager.setBeforeInstallSyncHook(null);
+ releaseWorker.countDown();
+ manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60));
+ try {
+ engine.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ });
+ }
+
+ /**
+ * An engine that owns its manager must use the whole-manager stop/join as
+ * its only quiescence barrier. Calling the per-ring barrier first would
+ * give a stuck worker two independent timeout budgets.
+ */
+ @Test(timeout = 30_000L)
+ public void testOwnedManagerCloseSkipsPerRingQuiescenceWait() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ String slot = tmpDir + "/owned-slot";
+ CursorSendEngine engine = new CursorSendEngine(slot, 4L * 1024 * 1024);
+ SegmentManager manager = readManager(engine);
+ AtomicBoolean perRingAwaited = new AtomicBoolean();
+ try {
+ manager.setBeforeRingQuiescenceAwaitHook(() -> perRingAwaited.set(true));
+ engine.close();
+ Assert.assertTrue("owned engine close did not complete", engine.isCloseCompleted());
+ Assert.assertFalse("owned engine close spent a separate per-ring wait budget",
+ perRingAwaited.get());
+ } finally {
+ manager.setBeforeRingQuiescenceAwaitHook(null);
+ engine.close();
+ }
+ });
+ }
+
+ /**
+ * Plain-positive path: after a normal close (worker quiesces promptly),
+ * a second engine must be able to acquire and use the same slot.
+ */
+ @Test(timeout = 30_000L)
+ public void testSameSlotReacquirableAfterNormalClose() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ String slot = tmpDir + "/slot";
+ CursorSendEngine first = new CursorSendEngine(slot, 4L * 1024 * 1024);
+ first.close();
+ CursorSendEngine second = new CursorSendEngine(slot, 4L * 1024 * 1024);
+ try {
+ Assert.assertFalse("fully-drained close must leave no segments to recover",
+ second.wasRecoveredFromDisk());
+ } finally {
+ second.close();
+ }
+ });
+ }
+
+ private static SegmentManager readManager(CursorSendEngine engine) throws Exception {
+ Field field = CursorSendEngine.class.getDeclaredField("manager");
+ field.setAccessible(true);
+ return (SegmentManager) field.get(engine);
+ }
+
+ private static SegmentRing readRing(CursorSendEngine engine) throws Exception {
+ Field field = CursorSendEngine.class.getDeclaredField("ring");
+ field.setAccessible(true);
+ return (SegmentRing) field.get(engine);
+ }
+
+ private static void rmDirRecursive(String dir) {
+ if (!Files.exists(dir)) return;
+ long find = Files.findFirst(dir);
+ if (find <= 0) return;
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ if (name != null && !".".equals(name) && !"..".equals(name)) {
+ String child = dir + "/" + name;
+ if (!Files.remove(child)) {
+ rmDirRecursive(child);
+ Files.remove(child);
+ }
+ }
+ rc = Files.findNext(find);
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
index f4de1ff2..c11fe1fe 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
@@ -197,6 +197,25 @@ public void testCloseIsIdempotent() throws Exception {
});
}
+ @Test
+ public void testCallbackCreationFailurePrecedesOwnedManagerResources() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ CursorSendEngine.setBeforeDeferredCloseCreationHook(() -> {
+ throw new OutOfMemoryError("simulated bound callback allocation failure");
+ });
+ try {
+ try {
+ new CursorSendEngine(tmpDir, 4096);
+ fail("expected callback allocation failure");
+ } catch (OutOfMemoryError expected) {
+ assertEquals("simulated bound callback allocation failure", expected.getMessage());
+ }
+ } finally {
+ CursorSendEngine.setBeforeDeferredCloseCreationHook(null);
+ }
+ });
+ }
+
@Test
public void testConstructorFailureAfterOwnedManagerStartCleansResources() throws Exception {
TestUtils.assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java
new file mode 100644
index 00000000..0a53e28e
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java
@@ -0,0 +1,256 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.DefaultHttpClientConfiguration;
+import io.questdb.client.cutlass.http.client.WebSocketClient;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.network.PlainSocketFactory;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class CursorWebSocketSendLoopBlockedSendCloseTest {
+
+ @Test(timeout = 30_000L)
+ public void testCloseBreaksBlockedSendBeforeJoiningWorker() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ BlockingSendClient client = new BlockingSendClient(true);
+ CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024);
+ CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
+ client,
+ engine,
+ 0L,
+ CursorWebSocketSendLoop.DEFAULT_PARK_NANOS,
+ null,
+ 100L,
+ 1_000L,
+ 5_000L,
+ false
+ );
+ long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT);
+ Thread closer = null;
+ AtomicReference closeFailure = new AtomicReference<>();
+ try {
+ Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L);
+ Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L);
+ Assert.assertEquals(0L, engine.appendBlocking(payload, 16));
+ loop.start();
+ Assert.assertTrue("I/O worker never entered the blocking send",
+ client.sendEntered.await(5, TimeUnit.SECONDS));
+
+ closer = new Thread(() -> {
+ try {
+ loop.close();
+ } catch (Throwable t) {
+ closeFailure.set(t);
+ }
+ }, "blocked-send-closer");
+ closer.start();
+
+ Assert.assertTrue("close did not break the traffic path before joining",
+ client.trafficClosed.await(5, TimeUnit.SECONDS));
+ Assert.assertEquals("traffic path must close exactly once", 1, client.trafficCloseCount.get());
+ Assert.assertEquals("the closer must break traffic, not the I/O worker",
+ closer, client.trafficCloseThread.get());
+ Assert.assertTrue("blocked send did not observe traffic-path closure",
+ client.sendExited.await(5, TimeUnit.SECONDS));
+
+ closer.join(TimeUnit.SECONDS.toMillis(5));
+ Assert.assertFalse("close did not join the I/O worker", closer.isAlive());
+ Assert.assertNull("close failed", closeFailure.get());
+ Assert.assertNull("ordinary close must not manufacture a terminal error", loop.getTerminalError());
+
+ Thread ioThread = client.sendThread.get();
+ Assert.assertNotNull(ioThread);
+ Assert.assertFalse("I/O worker must be dead when close returns", ioThread.isAlive());
+ Assert.assertEquals("full client cleanup must run exactly once", 1, client.cleanupCount.get());
+ Assert.assertEquals("the I/O worker must own cleanup before publishing exit",
+ ioThread, client.cleanupThread.get());
+ Assert.assertFalse("close must not manufacture caller interruption",
+ closer.isInterrupted());
+ } finally {
+ client.releaseSend.countDown();
+ if (closer != null) {
+ closer.join(TimeUnit.SECONDS.toMillis(5));
+ }
+ loop.close();
+ engine.close();
+ client.close();
+ Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT);
+ }
+ });
+ }
+
+ @Test(timeout = 30_000L)
+ public void testUnsupportedCustomTransportFailsWithoutDestroyingWorkerResources() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ BlockingSendClient client = new BlockingSendClient(false);
+ CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024);
+ CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
+ client,
+ engine,
+ 0L,
+ CursorWebSocketSendLoop.DEFAULT_PARK_NANOS,
+ null,
+ 100L,
+ 1_000L,
+ 5_000L,
+ false
+ );
+ long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT);
+ Thread closer = null;
+ AtomicReference closeFailure = new AtomicReference<>();
+ CountDownLatch closeEntered = new CountDownLatch(1);
+ try {
+ Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L);
+ Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L);
+ Assert.assertEquals(0L, engine.appendBlocking(payload, 16));
+ loop.start();
+ Assert.assertTrue("I/O worker never entered the blocking send",
+ client.sendEntered.await(5, TimeUnit.SECONDS));
+
+ closer = new Thread(() -> {
+ closeEntered.countDown();
+ try {
+ loop.close();
+ } catch (Throwable t) {
+ closeFailure.set(t);
+ }
+ }, "unsupported-transport-closer");
+ closer.start();
+
+ Assert.assertTrue("close did not start", closeEntered.await(5, TimeUnit.SECONDS));
+ closer.join(TimeUnit.SECONDS.toMillis(5));
+ Assert.assertFalse("unsupported transport made close join indefinitely", closer.isAlive());
+ Assert.assertTrue(closeFailure.get() instanceof LineSenderException);
+ Assert.assertTrue(closeFailure.get().getCause() instanceof UnsupportedOperationException);
+ Assert.assertEquals("unsupported cancellation must not release the blocked send",
+ 1L, client.sendExited.getCount());
+ Assert.assertEquals("unsupported cancellation must not perform full cleanup", 0, client.cleanupCount.get());
+ Assert.assertTrue("worker must retain resource ownership", client.sendThread.get().isAlive());
+
+ client.releaseSend.countDown();
+ Assert.assertTrue("released send did not exit", client.sendExited.await(5, TimeUnit.SECONDS));
+ Assert.assertTrue("worker did not complete delegated cleanup", client.cleanupDone.await(5, TimeUnit.SECONDS));
+ client.sendThread.get().join(TimeUnit.SECONDS.toMillis(5));
+ Assert.assertFalse("worker lingered after the custom transport was released", client.sendThread.get().isAlive());
+ Assert.assertEquals(1, client.cleanupCount.get());
+ Assert.assertNull("ordinary worker exit must remain non-terminal", loop.getTerminalError());
+ } finally {
+ client.releaseSend.countDown();
+ if (closer != null) {
+ closer.join(TimeUnit.SECONDS.toMillis(5));
+ }
+ Thread ioThread = client.sendThread.get();
+ if (ioThread != null) {
+ ioThread.join(TimeUnit.SECONDS.toMillis(5));
+ }
+ loop.close();
+ engine.close();
+ client.close();
+ Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT);
+ }
+ });
+ }
+
+ private static final class BlockingSendClient extends WebSocketClient {
+ private final AtomicBoolean cleanupClaimed = new AtomicBoolean();
+ private final AtomicInteger cleanupCount = new AtomicInteger();
+ private final CountDownLatch cleanupDone = new CountDownLatch(1);
+ private final AtomicReference cleanupThread = new AtomicReference<>();
+ private final boolean trafficShutdownSupported;
+ private final CountDownLatch releaseSend = new CountDownLatch(1);
+ private final CountDownLatch sendEntered = new CountDownLatch(1);
+ private final CountDownLatch sendExited = new CountDownLatch(1);
+ private final AtomicReference sendThread = new AtomicReference<>();
+ private final AtomicInteger trafficCloseCount = new AtomicInteger();
+ private final AtomicReference trafficCloseThread = new AtomicReference<>();
+ private final CountDownLatch trafficClosed = new CountDownLatch(1);
+
+ private BlockingSendClient(boolean trafficShutdownSupported) {
+ super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE);
+ this.trafficShutdownSupported = trafficShutdownSupported;
+ }
+
+ @Override
+ public void close() {
+ if (cleanupClaimed.compareAndSet(false, true)) {
+ cleanupCount.incrementAndGet();
+ cleanupThread.set(Thread.currentThread());
+ cleanupDone.countDown();
+ }
+ super.close();
+ }
+
+ @Override
+ public void closeTraffic() {
+ if (!trafficShutdownSupported) {
+ throw new UnsupportedOperationException("custom transport has no safe cancellation capability");
+ }
+ trafficCloseThread.compareAndSet(null, Thread.currentThread());
+ trafficCloseCount.incrementAndGet();
+ trafficClosed.countDown();
+ releaseSend.countDown();
+ }
+
+ @Override
+ public void sendBinary(long dataPtr, int length, int timeout) {
+ sendThread.set(Thread.currentThread());
+ sendEntered.countDown();
+ try {
+ if (!releaseSend.await(10, TimeUnit.SECONDS)) {
+ throw new AssertionError("traffic path was not closed");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("I/O worker was interrupted instead of traffic being closed", e);
+ } finally {
+ sendExited.countDown();
+ }
+ throw new LineSenderException("traffic path closed");
+ }
+
+ @Override
+ protected void ioWait(int timeout, int op) {
+ throw new UnsupportedOperationException("stub: no socket");
+ }
+
+ @Override
+ protected void setupIoWait() {
+ // no-op
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java
new file mode 100644
index 00000000..0225d071
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java
@@ -0,0 +1,469 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.DefaultHttpClientConfiguration;
+import io.questdb.client.cutlass.http.client.WebSocketClient;
+import io.questdb.client.cutlass.http.client.WebSocketFrameHandler;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.network.PlainSocketFactory;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * TRUE-CANCELLATION proof (P1): {@code close()} must cancel a connect attempt
+ * that is blocked inside {@code reconnectFactory.reconnect(...)} (a black-holed
+ * native connect that neither unpark nor interrupt can cancel). The in-flight
+ * {@link WebSocketClient} is walk-local -- it is NOT the loop's {@code client}
+ * field (that field is {@code null} on the async-initial connect and points at
+ * the stale pre-drop client on a mid-flight reconnect) -- so before the fix
+ * {@code close()}'s field-client {@code closeTraffic()} could not reach it and
+ * {@code close()} blocked on the untimed {@code shutdownLatch.await()} for the
+ * whole OS SYN-retry window (~60-130s per endpoint).
+ *
+ * The fix publishes the in-flight client to a race-safe
+ * {@link CursorWebSocketSendLoop.ConnectCancellation} handle before the
+ * blocking connect, and {@code close()} breaks that client's traffic. These
+ * tests assert TRUE cancellation, not just return: the fake in-flight client's
+ * {@code closeTraffic()} is the ONLY thing that unblocks the parked
+ * {@code reconnect()}, and the tests witness that {@code closeTraffic()} was
+ * invoked (exactly once, by the closer thread) AND that {@code close()}
+ * returned. Deterministic latches only; {@code @Test(timeout=...)} fails a
+ * regression fast.
+ */
+public class CursorWebSocketSendLoopConnectPhaseCloseTest {
+
+ /**
+ * Generous budget: with the fix, close() breaks the in-flight connect and
+ * returns well within this. Without the fix, close() blocks on the untimed
+ * shutdown latch for the entire (simulated) connect and this budget lapses.
+ */
+ private static final long CLOSE_BUDGET_MILLIS = 5_000L;
+
+ /**
+ * Small, injectable bounded-await backstop for the TOCTOU test. Shrunk from
+ * the production {@code DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS} (30 s) via
+ * {@link CursorWebSocketSendLoop#setShutdownAwaitTimeoutMillis(long)} so the
+ * timeout branch fires fast and deterministically -- no multi-second real
+ * wait -- while still leaving CLOSE_BUDGET_MILLIS of slack for the closer
+ * thread to return.
+ */
+ private static final long BACKSTOP_MILLIS = 500L;
+
+ /**
+ * Async-initial-connect path: the loop is built with a {@code null} client
+ * and a reconnect factory, so the I/O thread drives the very first connect
+ * through connectLoop while the {@code client} field stays {@code null}.
+ */
+ @Test(timeout = 30_000L)
+ public void testCloseCancelsAsyncInitialConnectViaInFlightClient() throws Exception {
+ runConnectCancelledByCloseTraffic(false);
+ }
+
+ /**
+ * Mid-flight-reconnect path: an initial client is installed, its first
+ * receive fails, so connectLoop reconnects and blocks. The {@code client}
+ * field then points at the stale pre-drop client, never the in-flight one.
+ */
+ @Test(timeout = 30_000L)
+ public void testCloseCancelsMidFlightReconnectViaInFlightClient() throws Exception {
+ runConnectCancelledByCloseTraffic(true);
+ }
+
+ /**
+ * BACKSTOP (bounded-await) proof: models the pathological, uninterruptible
+ * TOCTOU case where {@code cancel()}'s {@code closeTraffic()} CANNOT break
+ * the in-flight connect (as if cancellation landed after the pre-connect
+ * guard but before native fd creation, making closeTraffic() a no-op). The
+ * connect stays parked, so round-2 cancellation does NOT release the latch.
+ * Asserts {@code close()} STILL returns within the bounded backstop (does
+ * not hang) and surfaces the failed-stop contract: a loud
+ * {@link LineSenderException} whose message names the stalled I/O thread and
+ * the timeout, with client/engine teardown delegated to the I/O thread's
+ * own exit path (no destructive close under the still-live worker).
+ */
+ @Test(timeout = 30_000L)
+ public void testCloseReturnsBoundedWhenCancellationCannotBreakConnect() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ final AtomicReference published = new AtomicReference<>();
+ final CountDownLatch connectEntered = new CountDownLatch(1);
+ // The ONLY thing that releases the parked connect. closeTraffic()
+ // deliberately does NOT touch it, modelling a cancel that cannot
+ // break the connect; the test itself counts it down, AFTER the
+ // backstop assertions, so the worker can unwind for the leak check.
+ final CountDownLatch release = new CountDownLatch(1);
+
+ final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() {
+ @Override
+ public WebSocketClient reconnect() throws Exception {
+ return reconnect(null);
+ }
+
+ @Override
+ public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) {
+ UninterruptibleInFlightClient c = new UninterruptibleInFlightClient(release);
+ published.set(c);
+ if (cancellation != null) {
+ cancellation.publish(c);
+ }
+ connectEntered.countDown();
+ // Uninterruptible AND uncancellable: closeTraffic() does not
+ // release this park; only the test's release latch does.
+ c.awaitRelease();
+ c.close();
+ throw new LineSenderException("connect ended after release");
+ }
+ };
+
+ final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024);
+ final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
+ null,
+ engine,
+ 0L,
+ CursorWebSocketSendLoop.DEFAULT_PARK_NANOS,
+ factory,
+ /* reconnectMaxDurationMillis */ 60_000L,
+ /* reconnectInitialBackoffMillis */ 1_000L,
+ /* reconnectMaxBackoffMillis */ 5_000L,
+ false
+ );
+ // Shrink the bounded-await backstop so the timeout branch fires fast
+ // (no multi-second real wait); production uses the 30s default.
+ loop.setShutdownAwaitTimeoutMillis(BACKSTOP_MILLIS);
+
+ final AtomicReference closeFailure = new AtomicReference<>();
+ Thread closer = null;
+ try {
+ loop.start();
+ Assert.assertTrue("I/O worker never entered the blocking connect",
+ connectEntered.await(5, TimeUnit.SECONDS));
+
+ closer = new Thread(() -> {
+ try {
+ loop.close();
+ } catch (Throwable t) {
+ closeFailure.set(t);
+ }
+ }, "backstop-closer");
+ closer.start();
+
+ closer.join(CLOSE_BUDGET_MILLIS);
+ final boolean closedInTime = !closer.isAlive();
+
+ final UninterruptibleInFlightClient inFlight = published.get();
+ Assert.assertNotNull("no in-flight client was ever published", inFlight);
+ // cancel() DID attempt to break the connect -- but closeTraffic()
+ // is a no-op here, so the connect is still parked.
+ Assert.assertTrue("close() must have attempted to cancel the in-flight connect",
+ inFlight.trafficCloseCount.get() >= 1);
+ Assert.assertEquals("the parked connect must NOT have been released by closeTraffic()",
+ 1L, release.getCount());
+
+ // Decisive: close() returns within the bounded backstop even
+ // though cancellation could not break the connect.
+ Assert.assertTrue(
+ "close() must return within the bounded backstop (" + CLOSE_BUDGET_MILLIS
+ + "ms) even when cancellation cannot break the connect; instead it hung",
+ closedInTime);
+
+ // Failed-stop contract: loud LineSenderException naming the
+ // stalled thread and the timeout.
+ final Throwable failure = closeFailure.get();
+ Assert.assertNotNull("close() must loud-fail when the backstop times out", failure);
+ Assert.assertTrue("failed stop must be a LineSenderException, was " + failure,
+ failure instanceof LineSenderException);
+ Assert.assertTrue("message must name the stalled I/O thread: " + failure.getMessage(),
+ failure.getMessage().contains("cursor I/O thread did not stop"));
+ Assert.assertTrue("message must attribute the failed stop to the backstop timeout: "
+ + failure.getMessage(),
+ failure.getMessage().contains("timed out"));
+ // Teardown is delegated to the I/O thread's exit path, not done
+ // destructively under the live worker: no terminal manufactured.
+ Assert.assertNull("backstop timeout must not manufacture a terminal error",
+ loop.getTerminalError());
+ } finally {
+ // Now let the parked connect unwind so the worker exits and the
+ // leak check sees a clean teardown. Restore a generous backstop
+ // so the reconciling close() below awaits the worker's exit.
+ loop.setShutdownAwaitTimeoutMillis(CLOSE_BUDGET_MILLIS);
+ release.countDown();
+ if (closer != null) {
+ closer.join(TimeUnit.SECONDS.toMillis(5));
+ }
+ loop.close();
+ engine.close();
+ }
+ });
+ }
+
+ private void runConnectCancelledByCloseTraffic(boolean withInitialClient) throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ // Allocate the initial client INSIDE the leak-checked block so its
+ // native buffers are inside the baseline, not counted as over-free.
+ final WebSocketClient initialClient = withInitialClient ? new FailingInitialClient() : null;
+ final AtomicReference published = new AtomicReference<>();
+ final CountDownLatch connectEntered = new CountDownLatch(1);
+
+ // A reconnect factory that models the real connect walk: it creates
+ // the client it is about to block on, PUBLISHES it to the loop's
+ // cancellation handle before the blocking connect, then parks in a
+ // way that ONLY closeTraffic() (i.e. cancellation) can release --
+ // exactly a black-holed native connect that unpark/interrupt cannot
+ // cancel.
+ final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() {
+ @Override
+ public WebSocketClient reconnect() throws Exception {
+ return reconnect(null);
+ }
+
+ @Override
+ public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) {
+ InFlightConnectClient c = new InFlightConnectClient();
+ published.set(c);
+ if (cancellation != null) {
+ cancellation.publish(c);
+ }
+ connectEntered.countDown();
+ // Black-holed connect: returns only once closeTraffic() breaks it.
+ c.awaitTrafficBreak();
+ // Cancelled: dispose the half-built client (frees native
+ // buffers) and surface a transport error, exactly as the
+ // real connect walk's catch does on a broken connect.
+ c.close();
+ throw new LineSenderException("connect cancelled by closeTraffic()");
+ }
+ };
+
+ final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024);
+ final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
+ initialClient,
+ engine,
+ 0L,
+ CursorWebSocketSendLoop.DEFAULT_PARK_NANOS,
+ factory,
+ /* reconnectMaxDurationMillis */ 60_000L,
+ /* reconnectInitialBackoffMillis */ 1_000L,
+ /* reconnectMaxBackoffMillis */ 5_000L,
+ false
+ );
+
+ final AtomicReference closeFailure = new AtomicReference<>();
+ Thread closer = null;
+ try {
+ loop.start();
+ Assert.assertTrue("I/O worker never entered the blocking connect",
+ connectEntered.await(5, TimeUnit.SECONDS));
+
+ closer = new Thread(() -> {
+ try {
+ loop.close();
+ } catch (Throwable t) {
+ closeFailure.set(t);
+ }
+ }, "connect-phase-closer");
+ closer.start();
+
+ closer.join(CLOSE_BUDGET_MILLIS);
+ final boolean closedInTime = !closer.isAlive();
+
+ final InFlightConnectClient inFlight = published.get();
+ Assert.assertNotNull("no in-flight client was ever published to the cancellation handle",
+ inFlight);
+
+ // Let the worker finish unwinding regardless of outcome so the
+ // memory-leak check sees a clean teardown.
+ inFlight.trafficBroken.countDown();
+ closer.join(TimeUnit.SECONDS.toMillis(10));
+
+ Assert.assertNull("close() must not fail", closeFailure.get());
+ Assert.assertTrue(
+ "close() must cancel the connect blocked inside reconnect() and return "
+ + "within " + CLOSE_BUDGET_MILLIS + "ms; instead it blocked on the "
+ + "untimed shutdown latch for the whole connect",
+ closedInTime);
+ // Decisive: closeTraffic() on the IN-FLIGHT client is what
+ // unblocked the parked connect -- and it was the closer thread,
+ // not the I/O worker, that broke it.
+ Assert.assertEquals(
+ "close() must break the in-flight connect's traffic exactly once",
+ 1, inFlight.trafficCloseCount.get());
+ Assert.assertEquals(
+ "the closer thread (not the I/O worker) must cancel the in-flight connect",
+ closer, inFlight.trafficCloseThread.get());
+ Assert.assertNull("ordinary connect-phase close must not manufacture a terminal error",
+ loop.getTerminalError());
+ } finally {
+ InFlightConnectClient inFlight = published.get();
+ if (inFlight != null) {
+ inFlight.trafficBroken.countDown();
+ }
+ if (closer != null) {
+ closer.join(TimeUnit.SECONDS.toMillis(5));
+ }
+ loop.close();
+ engine.close();
+ if (initialClient != null) {
+ initialClient.close();
+ }
+ }
+ });
+ }
+
+ /**
+ * The fake in-flight client. It never really connects: the factory parks in
+ * {@link #awaitTrafficBreak()} until {@link #closeTraffic()} releases it,
+ * which is the ONLY path out -- neither unpark nor interrupt can cancel it.
+ */
+ private static final class InFlightConnectClient extends WebSocketClient {
+ final CountDownLatch trafficBroken = new CountDownLatch(1);
+ final AtomicInteger trafficCloseCount = new AtomicInteger();
+ final AtomicReference trafficCloseThread = new AtomicReference<>();
+
+ private InFlightConnectClient() {
+ super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE);
+ }
+
+ @Override
+ public void closeTraffic() {
+ trafficCloseThread.compareAndSet(null, Thread.currentThread());
+ trafficCloseCount.incrementAndGet();
+ trafficBroken.countDown();
+ }
+
+ void awaitTrafficBreak() {
+ // Uninterruptible: only closeTraffic()'s countDown may release us,
+ // faithfully modelling a native connect that unpark/interrupt cannot
+ // cancel.
+ boolean interrupted = false;
+ while (trafficBroken.getCount() != 0L) {
+ try {
+ trafficBroken.await();
+ } catch (InterruptedException e) {
+ interrupted = true;
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ protected void ioWait(int timeout, int op) {
+ throw new UnsupportedOperationException("stub: no socket");
+ }
+
+ @Override
+ protected void setupIoWait() {
+ // no-op
+ }
+ }
+
+ /**
+ * The fake in-flight client for the BACKSTOP test. Its {@link #closeTraffic()}
+ * is a no-op w.r.t. releasing the park -- it only records that cancellation
+ * was attempted -- so a {@code close()}->{@code cancel()} CANNOT unblock the
+ * parked connect. This models the pathological uninterruptible/TOCTOU case;
+ * the ONLY release is the test-owned {@code release} latch, counted down
+ * AFTER the backstop assertions.
+ */
+ private static final class UninterruptibleInFlightClient extends WebSocketClient {
+ final AtomicInteger trafficCloseCount = new AtomicInteger();
+ private final CountDownLatch release;
+
+ private UninterruptibleInFlightClient(CountDownLatch release) {
+ super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE);
+ this.release = release;
+ }
+
+ @Override
+ public void closeTraffic() {
+ // Records the cancel attempt but does NOT release the parked
+ // connect: exactly a closeTraffic() that lands before the native fd
+ // exists (a no-op), so the connect blocks on regardless.
+ trafficCloseCount.incrementAndGet();
+ }
+
+ void awaitRelease() {
+ // Uninterruptible: neither unpark, interrupt, nor closeTraffic()
+ // releases us -- only the test's release latch.
+ boolean interrupted = false;
+ while (release.getCount() != 0L) {
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ interrupted = true;
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ protected void ioWait(int timeout, int op) {
+ throw new UnsupportedOperationException("stub: no socket");
+ }
+
+ @Override
+ protected void setupIoWait() {
+ // no-op
+ }
+ }
+
+ /**
+ * A pre-installed client whose first receive fails, driving the I/O loop
+ * into a reconnect so the in-flight client is published while the
+ * {@code client} field still points here (the mid-flight-reconnect path).
+ */
+ private static final class FailingInitialClient extends WebSocketClient {
+
+ private FailingInitialClient() {
+ super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE);
+ }
+
+ @Override
+ public boolean tryReceiveFrame(WebSocketFrameHandler handler) {
+ throw new LineSenderException("initial wire dropped");
+ }
+
+ @Override
+ protected void ioWait(int timeout, int op) {
+ throw new UnsupportedOperationException("stub: no socket");
+ }
+
+ @Override
+ protected void setupIoWait() {
+ // no-op
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java
new file mode 100644
index 00000000..0c28ea16
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java
@@ -0,0 +1,255 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock;
+import io.questdb.client.std.Files;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.nio.file.Paths;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Regression test for the completion/release ordering in
+ * {@code CursorSendEngine.finishClose()}: {@code closeCompleted} must be
+ * published strictly AFTER the slot flock release is confirmed, never
+ * before.
+ *
+ *
The bug being pinned: {@code closeCompleted = true} used to be written
+ * before {@code slotLock} was closed. A pool thread could observe completion
+ * through {@code QwpWebSocketSender.isSlotLockReleased()}, free the slot
+ * index in {@code SenderPool.reprobeRetiredSlots()}, and admit a replacement
+ * sender whose {@code SlotLock.acquire} then collided with the still-open
+ * flock fd — a spurious "sf slot already in use" construction failure naming
+ * the process's own pid as the holder.
+ *
+ *
The window between the publish and the {@code Files.close(fd)} is
+ * microseconds wide in the wild; {@code setBeforeFlockReleaseHook} makes it
+ * deterministic. The closing thread is parked between terminal cleanup and
+ * the flock release, and the test asserts from outside the window's two
+ * halves of the contract:
+ *
+ *
inside the window: {@code isCloseCompleted()} is still false AND a
+ * fresh {@code SlotLock.acquire} on the slot fails (proving the flock
+ * is genuinely held — i.e. reporting completion here would have been
+ * a lie);
+ *
after the window: {@code isCloseCompleted()} is true AND a fresh
+ * {@code SlotLock.acquire} succeeds (completion still implies
+ * reusability — the reorder did not break the happy path).
+ *
+ */
+public class EngineClosePublishAfterFlockReleaseTest {
+
+ private String sfDir;
+
+ @Before
+ public void setUp() {
+ sfDir = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-engine-close-publish-order-" + System.nanoTime()).toString();
+ }
+
+ @After
+ public void tearDown() {
+ if (sfDir == null) return;
+ long find = Files.findFirst(sfDir);
+ if (find > 0) {
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ if (name != null && !".".equals(name) && !"..".equals(name)) {
+ Files.remove(sfDir + "/" + name);
+ }
+ rc = Files.findNext(find);
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ }
+ Files.remove(sfDir);
+ }
+
+ @Test(timeout = 30_000L)
+ public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024);
+ CountDownLatch inWindow = new CountDownLatch(1);
+ CountDownLatch proceed = new CountDownLatch(1);
+ engine.setBeforeFlockReleaseHook(() -> {
+ inWindow.countDown();
+ try {
+ // Bounded so a failed assertion on the main thread can
+ // never wedge the closer past the test timeout.
+ proceed.await(20, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+
+ AtomicReference closerError = new AtomicReference<>();
+ Thread closer = new Thread(() -> {
+ try {
+ engine.close();
+ } catch (Throwable t) {
+ closerError.set(t);
+ }
+ }, "engine-closer");
+ closer.start();
+
+ try {
+ assertTrue("closer thread never reached the cleanup/release window",
+ inWindow.await(10, TimeUnit.SECONDS));
+
+ // Inside the window: terminal cleanup has run, the flock has
+ // NOT been released. Completion must not be observable yet —
+ // this is the exact read a pool thread performs before
+ // freeing the slot index.
+ assertFalse("closeCompleted was published before the flock release; "
+ + "a pool observing this would free the slot index and a "
+ + "replacement sender would collide with the still-held flock",
+ engine.isCloseCompleted());
+
+ // Prove the window is real: the flock is genuinely still
+ // held, so acquisition by a "replacement engine" fails.
+ try {
+ SlotLock probe = SlotLock.acquire(sfDir);
+ probe.close();
+ fail("scaffolding error: expected the slot flock to still be "
+ + "held inside the pre-release window, but a fresh "
+ + "SlotLock.acquire succeeded");
+ } catch (IllegalStateException expected) {
+ // good — slot is locked, which is why completion must
+ // not have been published yet.
+ }
+ } finally {
+ proceed.countDown();
+ closer.join(10_000L);
+ }
+ assertFalse("closer thread did not finish", closer.isAlive());
+ assertNull("engine.close() threw", closerError.get());
+
+ // After the window: the release is confirmed, so completion must
+ // now be latched, and completion must still imply reusability.
+ assertTrue("closeCompleted must latch once the flock release is confirmed",
+ engine.isCloseCompleted());
+ try (SlotLock ignored = SlotLock.acquire(sfDir)) {
+ // good — completion implies the slot dir is acquirable.
+ } catch (IllegalStateException stillHeld) {
+ fail("closeCompleted reported true but the slot flock is still held: "
+ + stillHeld.getMessage());
+ }
+ });
+ }
+
+ /**
+ * The error half of the publish-after-release contract: when
+ * {@code SlotLock.release()} reports failure (the OS refused the explicit
+ * unlock, so the flock may still be held), {@code closeCompleted} must
+ * stay false until a later close confirms release. A pool observing
+ * {@code isCloseCompleted() == false} keeps the slot retired while the
+ * flock is genuinely held, then recovers it once the retry completes.
+ *
+ * The lock's fd is swapped to a known-bad descriptor for a deterministic
+ * native unlock failure, then restored before retrying through the engine's
+ * public close API.
+ */
+ @Test(timeout = 30_000L)
+ public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024);
+ Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock");
+ slotLockField.setAccessible(true);
+ SlotLock slotLock = (SlotLock) slotLockField.get(engine);
+ assertNotNull("disk-mode engine must hold a slot lock", slotLock);
+ Field fdField = SlotLock.class.getDeclaredField("fd");
+ fdField.setAccessible(true);
+ int realFd = fdField.getInt(slotLock);
+ assertTrue("precondition: live flock fd", realFd >= 0);
+
+ // This test owns the explicit close() retry. Prevent the shared
+ // asynchronous retry driver from racing it and from surviving into
+ // another test class; FlockReleaseRetryDriverTest covers the real
+ // driver lifecycle separately. A start failure synchronously clears
+ // the shared retry queue and resets the engine's scheduling flag.
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> {
+ throw new IllegalStateException("retry driver disabled for explicit close retry test");
+ });
+ try {
+ // A non-negative fd no process has open: close(2) fails EBADF,
+ // so finishClose's release() confirmation fails.
+ fdField.setInt(slotLock, 1_000_000_000);
+ engine.close();
+ assertFalse(
+ "closeCompleted was published despite an unconfirmed flock release; "
+ + "a pool observing this would free the slot index while the "
+ + "flock fd is still open",
+ engine.isCloseCompleted());
+ // The REAL flock is still held — the incomplete report is true.
+ try {
+ SlotLock probe = SlotLock.acquire(sfDir);
+ probe.close();
+ fail("slot must not be acquirable while the original flock fd is still open");
+ } catch (IllegalStateException expected) {
+ // good — incomplete close really means "still locked".
+ }
+
+ // Remove the injected fault. The retry must only re-attempt
+ // the retained fd release: ring/watermark cleanup stays
+ // exactly-once, while completion and slot reusability recover.
+ fdField.setInt(slotLock, realFd);
+ engine.close();
+ assertTrue("retried close() must complete after the flock release succeeds",
+ engine.isCloseCompleted());
+ try (SlotLock ignored = SlotLock.acquire(sfDir)) {
+ // good — eventual completion implies reusable capacity.
+ }
+ } finally {
+ try {
+ // If an assertion failed before the successful retry, restore
+ // and release the real fd so the test never leaks a flock.
+ if (!engine.isCloseCompleted()) {
+ fdField.setInt(slotLock, realFd);
+ assertTrue("restored fd must release cleanly", slotLock.release());
+ }
+ } finally {
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(null);
+ }
+ }
+ });
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java
new file mode 100644
index 00000000..0037437f
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java
@@ -0,0 +1,508 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock;
+import io.questdb.client.std.Files;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class FlockReleaseRetryDriverTest {
+
+ private final List sfDirs = new ArrayList<>();
+
+ @After
+ public void tearDown() {
+ CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null);
+ CursorSendEngine.setFlockReleaseRetryParkOverride(null);
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(null);
+ for (String sfDir : sfDirs) {
+ removeDir(sfDir);
+ }
+ }
+
+ /**
+ * Driver-start failure must not strand retired capacity until process
+ * exit. {@code Sender.close()} is a one-shot no-op by contract, so the
+ * only recovery surface a pool has is its retired-slot probe
+ * ({@code isSlotLockReleased()}, called from the housekeeper tick and
+ * capacity-starved borrows) — that probe must re-arm the shared retry
+ * driver once thread creation works again.
+ */
+ @Test(timeout = 30_000L)
+ public void testDriverStartFailureRecoversViaPoolProbe() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ AtomicInteger starts = new AtomicInteger();
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) {
+ @Override
+ public synchronized void start() {
+ starts.incrementAndGet();
+ throw new IllegalStateException("injected start failure");
+ }
+ });
+
+ CursorSendEngine engine = new CursorSendEngine(
+ newSfDir("probe-rearm"), 4L * 1024 * 1024);
+ SlotLock slotLock = slotLock(engine);
+ int realFd = fd(slotLock);
+ QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1);
+ sender.setCursorEngine(engine, true);
+ AtomicReference rearmedDriver = new AtomicReference<>();
+ boolean recovered = false;
+ try {
+ setFd(slotLock, 1_000_000_000);
+ sender.close();
+ assertEquals("retry driver start must be attempted once", 1, starts.get());
+ assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted());
+
+ // Sender.close() is idempotent: repeat calls never reach the
+ // engine again, so they cannot restart the failed driver.
+ sender.close();
+ assertEquals("no-op close must not retry the driver start", 1, starts.get());
+
+ // While the fault persists, each pool probe re-attempts the
+ // re-arm (and fails again) without publishing a release.
+ assertFalse("failed unlock must keep the slot reported as held",
+ sender.isSlotLockReleased());
+ assertTrue("pool probe must re-attempt the driver start",
+ starts.get() >= 2);
+
+ // The transient condition clears: thread creation works again
+ // (the failed start drained the queue, so the factory swap is
+ // legal) and the flock fd is back.
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> {
+ Thread thread = new Thread(task, "test-rearmed-flock-release-retry");
+ rearmedDriver.set(thread);
+ return thread;
+ });
+ setFd(slotLock, realFd);
+ CountDownLatch released = new CountDownLatch(1);
+ engine.setSlotLockReleaseListener(released::countDown);
+
+ // Production recovery surface: the pool re-probes the retired
+ // slot through isSlotLockReleased().
+ sender.isSlotLockReleased();
+ assertTrue("pool probe must re-arm the flock-release retry after "
+ + "a driver start failure",
+ released.await(10, TimeUnit.SECONDS));
+ assertTrue(engine.isCloseCompleted());
+ assertTrue("probe must expose the recovered release",
+ sender.isSlotLockReleased());
+ recovered = true;
+ } finally {
+ Thread driver = rearmedDriver.get();
+ if (driver != null) {
+ driver.join(10_000L);
+ }
+ if (!recovered) {
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(null);
+ setFd(slotLock, realFd);
+ if (!slotLock.release()) {
+ fail("restored flock fd did not release");
+ }
+ }
+ }
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(null);
+ });
+ }
+
+ @Test(timeout = 30_000L)
+ public void testPersistentFailuresShareOneRetryThread() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ final int engineCount = 32;
+ AtomicInteger retryFailures = new AtomicInteger();
+ AtomicInteger threadsCreated = new AtomicInteger();
+ AtomicReference retryThreadRef = new AtomicReference<>();
+ CountDownLatch retryFailuresObserved = new CountDownLatch(engineCount * 2);
+ CountDownLatch retryThreadStarted = new CountDownLatch(1);
+ CountDownLatch runRetryDriver = new CountDownLatch(1);
+ CursorSendEngine.setAfterFlockReleaseRetryFailureHook(() -> {
+ retryFailures.incrementAndGet();
+ retryFailuresObserved.countDown();
+ });
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> {
+ threadsCreated.incrementAndGet();
+ Thread thread = new Thread(() -> {
+ retryThreadStarted.countDown();
+ try {
+ runRetryDriver.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ task.run();
+ }, "test-shared-flock-release-retry");
+ retryThreadRef.set(thread);
+ return thread;
+ });
+
+ List engines = new ArrayList<>();
+ List slotLocks = new ArrayList<>();
+ List realFds = new ArrayList<>();
+ boolean fdsRestored = false;
+ try {
+ for (int i = 0; i < engineCount; i++) {
+ String sfDir = newSfDir("persistent-" + i);
+ CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024);
+ SlotLock slotLock = slotLock(engine);
+ int realFd = fd(slotLock);
+ engines.add(engine);
+ slotLocks.add(slotLock);
+ realFds.add(realFd);
+ setFd(slotLock, 1_000_000_000);
+ engine.close();
+ assertFalse("injected unlock failure must keep close incomplete",
+ engine.isCloseCompleted());
+ if (i == 0) {
+ assertTrue("retry thread was not started",
+ retryThreadStarted.await(10, TimeUnit.SECONDS));
+ }
+ }
+
+ assertEquals("persistent failures must share one retry thread",
+ 1, threadsCreated.get());
+ runRetryDriver.countDown();
+ assertTrue("driver did not perform two failed rounds",
+ retryFailuresObserved.await(10, TimeUnit.SECONDS));
+ assertTrue("driver did not retain persistent failures",
+ retryFailures.get() >= engineCount * 2);
+ for (CursorSendEngine engine : engines) {
+ assertFalse("failed releases must remain unpublished",
+ engine.isCloseCompleted());
+ }
+
+ restoreFds(slotLocks, realFds);
+ fdsRestored = true;
+ Thread retryThread = retryThreadRef.get();
+ retryThread.join(10_000L);
+ assertFalse("shared retry thread retained lifecycle resources after drain",
+ retryThread.isAlive());
+ for (CursorSendEngine engine : engines) {
+ assertTrue("driver must release every restored flock",
+ engine.isCloseCompleted());
+ }
+ assertEquals("retries must not create another thread",
+ 1, threadsCreated.get());
+ } finally {
+ if (!fdsRestored) {
+ restoreFds(slotLocks, realFds);
+ }
+ runRetryDriver.countDown();
+ Thread retryThread = retryThreadRef.get();
+ if (retryThread != null) {
+ retryThread.join(10_000L);
+ }
+ CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null);
+ }
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(null);
+ });
+ }
+
+ /**
+ * Schedule inspection for the shared driver's retry cadence: the
+ * inter-round park must grow exponentially from 100 ms and cap at 5 s,
+ * so a persistent unlock failure does not burn 10 syscalls per second
+ * per engine forever. The park override replaces the real park, so the
+ * test coordinates rounds without wall-clock waits.
+ */
+ @Test(timeout = 30_000L)
+ public void testRetryBackoffDoublesToCap() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ List parks = new ArrayList<>();
+ Semaphore parked = new Semaphore(0);
+ Semaphore proceed = new Semaphore(0);
+ AtomicReference driverRef = new AtomicReference<>();
+ CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> {
+ parks.add(nanos);
+ parked.release();
+ proceed.acquireUninterruptibly();
+ });
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> {
+ Thread thread = new Thread(task, "test-backoff-flock-release-retry");
+ driverRef.set(thread);
+ return thread;
+ });
+
+ CursorSendEngine engine = new CursorSendEngine(
+ newSfDir("backoff-cap"), 4L * 1024 * 1024);
+ SlotLock slotLock = slotLock(engine);
+ int realFd = fd(slotLock);
+ boolean fdRestored = false;
+ try {
+ setFd(slotLock, 1_000_000_000);
+ engine.close();
+ assertFalse("injected unlock failure must keep close incomplete",
+ engine.isCloseCompleted());
+
+ // Eight failed rounds: enough to observe the full ramp and
+ // two capped parks.
+ for (int round = 1; round <= 8; round++) {
+ assertTrue("driver did not park after failed round " + round,
+ parked.tryAcquire(10, TimeUnit.SECONDS));
+ if (round < 8) {
+ proceed.release();
+ }
+ }
+
+ setFd(slotLock, realFd);
+ fdRestored = true;
+ proceed.release();
+ Thread driver = driverRef.get();
+ driver.join(10_000L);
+ assertFalse("driver did not drain after the release succeeded",
+ driver.isAlive());
+ assertTrue("restored flock must be released", engine.isCloseCompleted());
+
+ List expected = new ArrayList<>();
+ expected.add(100_000_000L);
+ expected.add(200_000_000L);
+ expected.add(400_000_000L);
+ expected.add(800_000_000L);
+ expected.add(1_600_000_000L);
+ expected.add(3_200_000_000L);
+ expected.add(5_000_000_000L);
+ expected.add(5_000_000_000L);
+ assertEquals("retry parks must double from 100ms and cap at 5s",
+ expected, parks);
+ } finally {
+ if (!fdRestored) {
+ setFd(slotLock, realFd);
+ }
+ proceed.release(1_000);
+ Thread driver = driverRef.get();
+ if (driver != null) {
+ driver.join(10_000L);
+ }
+ }
+ CursorSendEngine.setFlockReleaseRetryParkOverride(null);
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(null);
+ });
+ }
+
+ /**
+ * A successful release in a round is progress: the driver must reset its
+ * backoff to the base so the remaining engines are retried promptly
+ * while the failure condition is clearing.
+ */
+ @Test(timeout = 30_000L)
+ public void testRetryBackoffResetsOnProgress() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ List parks = new ArrayList<>();
+ Semaphore parked = new Semaphore(0);
+ Semaphore proceed = new Semaphore(0);
+ AtomicReference driverRef = new AtomicReference<>();
+ CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> {
+ parks.add(nanos);
+ parked.release();
+ proceed.acquireUninterruptibly();
+ });
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> {
+ Thread thread = new Thread(task, "test-reset-flock-release-retry");
+ driverRef.set(thread);
+ return thread;
+ });
+
+ CursorSendEngine engineA = new CursorSendEngine(
+ newSfDir("backoff-reset-a"), 4L * 1024 * 1024);
+ CursorSendEngine engineB = new CursorSendEngine(
+ newSfDir("backoff-reset-b"), 4L * 1024 * 1024);
+ SlotLock slotLockA = slotLock(engineA);
+ SlotLock slotLockB = slotLock(engineB);
+ int realFdA = fd(slotLockA);
+ int realFdB = fd(slotLockB);
+ boolean fdARestored = false;
+ boolean fdBRestored = false;
+ try {
+ setFd(slotLockA, 1_000_000_000);
+ setFd(slotLockB, 1_000_000_001);
+ engineA.close();
+ engineB.close();
+
+ // Three failed rounds ramp the backoff to 400ms.
+ for (int round = 1; round <= 3; round++) {
+ assertTrue("driver did not park after failed round " + round,
+ parked.tryAcquire(10, TimeUnit.SECONDS));
+ if (round < 3) {
+ proceed.release();
+ }
+ }
+
+ // Engine A recovers; round 4 has one success and one failure,
+ // so its park must be back at the 100ms base.
+ setFd(slotLockA, realFdA);
+ fdARestored = true;
+ proceed.release();
+ assertTrue("driver did not park after the mixed round",
+ parked.tryAcquire(10, TimeUnit.SECONDS));
+ assertTrue("recovered engine must publish completion",
+ engineA.isCloseCompleted());
+
+ setFd(slotLockB, realFdB);
+ fdBRestored = true;
+ proceed.release();
+ Thread driver = driverRef.get();
+ driver.join(10_000L);
+ assertFalse("driver did not drain after both releases succeeded",
+ driver.isAlive());
+ assertTrue(engineB.isCloseCompleted());
+
+ List expected = new ArrayList<>();
+ expected.add(100_000_000L);
+ expected.add(200_000_000L);
+ expected.add(400_000_000L);
+ expected.add(100_000_000L);
+ assertEquals("a successful release must reset the backoff to base",
+ expected, parks);
+ } finally {
+ if (!fdARestored) {
+ setFd(slotLockA, realFdA);
+ }
+ if (!fdBRestored) {
+ setFd(slotLockB, realFdB);
+ }
+ proceed.release(1_000);
+ Thread driver = driverRef.get();
+ if (driver != null) {
+ driver.join(10_000L);
+ }
+ }
+ CursorSendEngine.setFlockReleaseRetryParkOverride(null);
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(null);
+ });
+ }
+
+ @Test(timeout = 30_000L)
+ public void testRetryThreadStartFailureLeavesExplicitCloseRetryable() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ AtomicInteger starts = new AtomicInteger();
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) {
+ @Override
+ public synchronized void start() {
+ starts.incrementAndGet();
+ throw new IllegalStateException("injected start failure");
+ }
+ });
+
+ CursorSendEngine engine = new CursorSendEngine(
+ newSfDir("start-failure"), 4L * 1024 * 1024);
+ SlotLock slotLock = slotLock(engine);
+ int realFd = fd(slotLock);
+ try {
+ setFd(slotLock, 1_000_000_000);
+ engine.close();
+ assertEquals("retry driver start must be attempted once", 1, starts.get());
+ assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted());
+
+ // This also proves the failed driver cleared its queue: the
+ // setter rejects replacement while any engine remains queued.
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(null);
+ setFd(slotLock, realFd);
+ engine.close();
+ assertTrue("explicit close must recover after retry-thread start failure",
+ engine.isCloseCompleted());
+ } finally {
+ if (!engine.isCloseCompleted()) {
+ CursorSendEngine.setFlockReleaseRetryThreadFactory(null);
+ setFd(slotLock, realFd);
+ if (!slotLock.release()) {
+ fail("restored flock fd did not release");
+ }
+ }
+ }
+ });
+ }
+
+ private static int fd(SlotLock slotLock) throws Exception {
+ Field fdField = SlotLock.class.getDeclaredField("fd");
+ fdField.setAccessible(true);
+ return fdField.getInt(slotLock);
+ }
+
+ private static void removeDir(String sfDir) {
+ long find = Files.findFirst(sfDir);
+ if (find > 0) {
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ if (name != null && !".".equals(name) && !"..".equals(name)) {
+ Files.remove(sfDir + "/" + name);
+ }
+ rc = Files.findNext(find);
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ }
+ Files.remove(sfDir);
+ }
+
+ private static void restoreFds(List slotLocks, List realFds) throws Exception {
+ for (int i = 0; i < slotLocks.size(); i++) {
+ SlotLock slotLock = slotLocks.get(i);
+ synchronized (slotLock) {
+ setFd(slotLock, realFds.get(i));
+ }
+ }
+ }
+
+ private static void setFd(SlotLock slotLock, int fd) throws Exception {
+ Field fdField = SlotLock.class.getDeclaredField("fd");
+ fdField.setAccessible(true);
+ fdField.setInt(slotLock, fd);
+ }
+
+ private static SlotLock slotLock(CursorSendEngine engine) throws Exception {
+ Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock");
+ slotLockField.setAccessible(true);
+ return (SlotLock) slotLockField.get(engine);
+ }
+
+ private String newSfDir(String suffix) {
+ String sfDir = Paths.get(
+ System.getProperty("java.io.tmpdir"),
+ "qdb-flock-release-retry-" + suffix + "-" + System.nanoTime()
+ ).toString();
+ sfDirs.add(sfDir);
+ return sfDir;
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
index 177ee5f6..cff8249b 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
@@ -128,7 +128,7 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception {
assertTrue(expected.getMessage(),
expected.getMessage().contains("pre-allocation failed"));
}
- assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls);
+ assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls);
assertEquals("allocate must run exactly once", 1, ff.allocateCalls);
assertEquals("fd must be closed on allocate failure", 1, ff.closeCalls);
assertEquals("file must be removed on allocate failure", 1, ff.removeCalls);
@@ -138,28 +138,31 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception {
}
@Test
- public void testCreateFailsCleanlyWhenOpenCleanRWReturnsMinusOne() throws Exception {
+ public void testCreateFailsCleanlyWhenExclusiveOpenReturnsMinusOne() throws Exception {
TestUtils.assertMemoryLeak(() -> {
String path = tmpDir + "/seg-noopen.sfa";
long sizeBytes = MmapSegment.HEADER_SIZE
+ MmapSegment.FRAME_HEADER_SIZE + 64;
FaultyFilesFacade ff = new FaultyFilesFacade();
- ff.failOnOpenCleanRW = true;
+ ff.failOnOpenRWExclusive = true;
try {
MmapSegment.create(ff, path, 0L, sizeBytes).close();
- fail("expected MmapSegmentException from openCleanRW returning -1");
+ fail("expected MmapSegmentException from openRWExclusive returning -1");
} catch (MmapSegmentException expected) {
assertTrue(expected.getMessage(),
- expected.getMessage().contains("openCleanRW failed"));
+ expected.getMessage().contains("exclusive create failed"));
}
- assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls);
- assertEquals("allocate must not run after openCleanRW failure",
+ assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls);
+ assertEquals("allocate must not run after exclusive-create failure",
0, ff.allocateCalls);
assertEquals("close must not be called when no fd was opened",
0, ff.closeCalls);
- assertEquals("remove must not be called when openCleanRW failed",
+ // With O_EXCL semantics a create failure can mean "path already
+ // exists and belongs to another lifecycle" -- create() must NOT
+ // unlink a file it never owned.
+ assertEquals("remove must not be called when exclusive create failed",
0, ff.removeCalls);
- assertFalse("no file should exist when openCleanRW failed",
+ assertFalse("no file should exist when exclusive create failed",
Files.exists(path));
});
}
@@ -198,7 +201,7 @@ public void testCreateRepeatedAllocateFailuresDoNotAccumulateOrphans() throws Ex
}
assertEquals("no orphan files may survive repeated allocate failures",
0, survivors);
- assertEquals(attempts, ff.openCleanRWCalls);
+ assertEquals(attempts, ff.openRWExclusiveCalls);
assertEquals(attempts, ff.allocateCalls);
assertEquals(attempts, ff.closeCalls);
assertEquals(attempts, ff.removeCalls);
@@ -495,9 +498,29 @@ private static final class FaultyFilesFacade implements FilesFacade {
int closeCalls;
boolean failOnAllocate;
boolean failOnOpenCleanRW;
+ boolean failOnOpenRWExclusive;
int openCleanRWCalls;
+ int openRWExclusiveCalls;
int removeCalls;
+ @Override
+ public int openRWExclusive(String path) {
+ openRWExclusiveCalls++;
+ if (failOnOpenRWExclusive) {
+ return -1;
+ }
+ return INSTANCE.openRWExclusive(path);
+ }
+
+ @Override
+ public int openRWExclusive(long pathPtr) {
+ openRWExclusiveCalls++;
+ if (failOnOpenRWExclusive) {
+ return -1;
+ }
+ return INSTANCE.openRWExclusive(pathPtr);
+ }
+
@Override
public long allocNativePath(String path) {
return INSTANCE.allocNativePath(path);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java
index 3d7c6a7c..cee8e41d 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java
@@ -204,6 +204,391 @@ public void testCloseDoesNotFreePathScratchWhenWorkerStillAlive() throws Excepti
});
}
+ /**
+ * Pins the claim-time registration gate at the top of
+ * {@code SegmentManager.serviceRing}: a snapshot entry whose ring was
+ * deregistered BEFORE the worker claims it must be skipped entirely —
+ * never claimed as {@code inService}, never handed to
+ * {@code serviceRing0}. This is the exact guarantee
+ * {@code CursorSendEngine.close()} relies on when it releases the ring,
+ * watermark and slot flock right after
+ * {@code awaitRingQuiescence(ring) == true}: the deregistering thread may
+ * already be freeing those resources, so a stale snapshot entry must not
+ * be touched at all (spare install, watermark write, drainTrimmable, or
+ * path building under the slot dir).
+ *
+ * Deterministic shape: three rings A, B, C registered before start, so
+ * the worker's first snapshot is exactly [A, B, C]. The worker parks in
+ * A's spare-install pass; while it is parked, B is deregistered (and a
+ * quiescence barrier for B must pass immediately — no pass for B is in
+ * flight). After release the worker walks the rest of its stale
+ * snapshot: it must skip B and service C. Every serviced ring is
+ * recorded from inside the worker's own pass (via the trim-sync hook
+ * reading {@code inService}), so the assertion is exact — no timing
+ * grace, no sleeps.
+ */
+ @Test(timeout = 15_000L)
+ public void testStaleSnapshotEntrySkippedAfterDeregisterBeforeServiceClaim() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32);
+ SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60));
+ SegmentRing[] rings = new SegmentRing[3];
+ String[] slots = new String[3];
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean fired = new AtomicBoolean();
+ AtomicReference hookErr = new AtomicReference<>();
+ // Rings the worker actually claimed and serviced, recorded from
+ // the worker thread itself at the trim-sync point every service
+ // pass reaches (spare needed or not).
+ java.util.Set